41ee5e8bglvqKvZSY5uJ5JGQejEwyQ linux-2.6.10-xen-sparse/drivers/xen/usbback/usbback.c
41ee5e8ckZ9xVNvu9NHIZDK7JqApmQ linux-2.6.10-xen-sparse/drivers/xen/usbfront/usbfront.c
41ee5e8ck9scpGirfqEZRARbGDyTXA linux-2.6.10-xen-sparse/drivers/xen/usbfront/xhci.h
+4236f620IqJ4VZVDPfMJzrpFrio8Sw linux-2.6.10-xen-sparse/fs/exec.c
412f47e4RKD-R5IS5gEXvcT8L4v8gA linux-2.6.10-xen-sparse/include/asm-generic/pgtable.h
40f56239YAjS52QG2FIAQpHDZAdGHg linux-2.6.10-xen-sparse/include/asm-xen/asm-i386/desc.h
4107adf1E5O4ztGHNGMzCCNhcvqNow linux-2.6.10-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h
4124f66f4NaKNa0xPiGGykn9QaZk3w linux-2.6.10-xen-sparse/include/linux/skbuff.h
419dfc6awx7w88wk6cG9P3mPidX6LQ linux-2.6.10-xen-sparse/kernel/irq/manage.c
40f56a0ddHCSs3501MY4hRf22tctOw linux-2.6.10-xen-sparse/mkbuildtree
+4236f620IaM-42pgVYuNGF4cFrttbw linux-2.6.10-xen-sparse/mm/highmem.c
412f46c0LJuKAgSPGoC0Z1DEkLfuLA linux-2.6.10-xen-sparse/mm/memory.c
410a94a4KT6I6X0LVc7djB39tRDp4g linux-2.6.10-xen-sparse/mm/page_alloc.c
+4236f620F2ZXlYSPUkwtN85tZMqDFQ linux-2.6.10-xen-sparse/mm/swapfile.c
41505c572m-s9ATiO1LiD1GPznTTIg linux-2.6.10-xen-sparse/net/core/skbuff.c
413cb1e4zst25MDYjg63Y-NGC5_pLg netbsd-2.0-xen-sparse/Makefile
413cb1e5c_Mkxf_X0zimEhTKI_l4DA netbsd-2.0-xen-sparse/mkbuildtree
error_code |= (regs->xcs & 2) << 1;
if (regs->eflags & X86_EFLAGS_VM)
error_code |= 4;
+
+#ifdef CONFIG_XEN_BATCH_MODE2
+ /* ensure all updates have completed */
+ flush_page_update_queue();
+#endif
+
if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
SIGSEGV) == NOTIFY_STOP)
--- /dev/null
+/*
+ * linux/fs/exec.c
+ *
+ * Copyright (C) 1991, 1992 Linus Torvalds
+ */
+
+/*
+ * #!-checking implemented by tytso.
+ */
+/*
+ * Demand-loading implemented 01.12.91 - no need to read anything but
+ * the header into memory. The inode of the executable is put into
+ * "current->executable", and page faults do the actual loading. Clean.
+ *
+ * Once more I can proudly say that linux stood up to being changed: it
+ * was less than 2 hours work to get demand-loading completely implemented.
+ *
+ * Demand loading changed July 1993 by Eric Youngdale. Use mmap instead,
+ * current->executable is only used by the procfs. This allows a dispatch
+ * table to check for several different types of binary formats. We keep
+ * trying until we recognize the file or we run out of supported binary
+ * formats.
+ */
+
+#include <linux/config.h>
+#include <linux/slab.h>
+#include <linux/file.h>
+#include <linux/mman.h>
+#include <linux/a.out.h>
+#include <linux/stat.h>
+#include <linux/fcntl.h>
+#include <linux/smp_lock.h>
+#include <linux/init.h>
+#include <linux/pagemap.h>
+#include <linux/highmem.h>
+#include <linux/spinlock.h>
+#include <linux/key.h>
+#include <linux/personality.h>
+#include <linux/binfmts.h>
+#include <linux/swap.h>
+#include <linux/utsname.h>
+#include <linux/module.h>
+#include <linux/namei.h>
+#include <linux/proc_fs.h>
+#include <linux/ptrace.h>
+#include <linux/mount.h>
+#include <linux/security.h>
+#include <linux/syscalls.h>
+#include <linux/rmap.h>
+
+#include <asm/uaccess.h>
+#include <asm/mmu_context.h>
+
+#ifdef CONFIG_KMOD
+#include <linux/kmod.h>
+#endif
+
+int core_uses_pid;
+char core_pattern[65] = "core";
+/* The maximal length of core_pattern is also specified in sysctl.c */
+
+static struct linux_binfmt *formats;
+static rwlock_t binfmt_lock = RW_LOCK_UNLOCKED;
+
+int register_binfmt(struct linux_binfmt * fmt)
+{
+ struct linux_binfmt ** tmp = &formats;
+
+ if (!fmt)
+ return -EINVAL;
+ if (fmt->next)
+ return -EBUSY;
+ write_lock(&binfmt_lock);
+ while (*tmp) {
+ if (fmt == *tmp) {
+ write_unlock(&binfmt_lock);
+ return -EBUSY;
+ }
+ tmp = &(*tmp)->next;
+ }
+ fmt->next = formats;
+ formats = fmt;
+ write_unlock(&binfmt_lock);
+ return 0;
+}
+
+EXPORT_SYMBOL(register_binfmt);
+
+int unregister_binfmt(struct linux_binfmt * fmt)
+{
+ struct linux_binfmt ** tmp = &formats;
+
+ write_lock(&binfmt_lock);
+ while (*tmp) {
+ if (fmt == *tmp) {
+ *tmp = fmt->next;
+ write_unlock(&binfmt_lock);
+ return 0;
+ }
+ tmp = &(*tmp)->next;
+ }
+ write_unlock(&binfmt_lock);
+ return -EINVAL;
+}
+
+EXPORT_SYMBOL(unregister_binfmt);
+
+static inline void put_binfmt(struct linux_binfmt * fmt)
+{
+ module_put(fmt->module);
+}
+
+/*
+ * Note that a shared library must be both readable and executable due to
+ * security reasons.
+ *
+ * Also note that we take the address to load from from the file itself.
+ */
+asmlinkage long sys_uselib(const char __user * library)
+{
+ struct file * file;
+ struct nameidata nd;
+ int error;
+
+ nd.intent.open.flags = FMODE_READ;
+ error = __user_walk(library, LOOKUP_FOLLOW|LOOKUP_OPEN, &nd);
+ if (error)
+ goto out;
+
+ error = -EINVAL;
+ if (!S_ISREG(nd.dentry->d_inode->i_mode))
+ goto exit;
+
+ error = permission(nd.dentry->d_inode, MAY_READ | MAY_EXEC, &nd);
+ if (error)
+ goto exit;
+
+ file = dentry_open(nd.dentry, nd.mnt, O_RDONLY);
+ error = PTR_ERR(file);
+ if (IS_ERR(file))
+ goto out;
+
+ error = -ENOEXEC;
+ if(file->f_op) {
+ struct linux_binfmt * fmt;
+
+ read_lock(&binfmt_lock);
+ for (fmt = formats ; fmt ; fmt = fmt->next) {
+ if (!fmt->load_shlib)
+ continue;
+ if (!try_module_get(fmt->module))
+ continue;
+ read_unlock(&binfmt_lock);
+ error = fmt->load_shlib(file);
+ read_lock(&binfmt_lock);
+ put_binfmt(fmt);
+ if (error != -ENOEXEC)
+ break;
+ }
+ read_unlock(&binfmt_lock);
+ }
+ fput(file);
+out:
+ return error;
+exit:
+ path_release(&nd);
+ goto out;
+}
+
+/*
+ * count() counts the number of strings in array ARGV.
+ */
+static int count(char __user * __user * argv, int max)
+{
+ int i = 0;
+
+ if (argv != NULL) {
+ for (;;) {
+ char __user * p;
+
+ if (get_user(p, argv))
+ return -EFAULT;
+ if (!p)
+ break;
+ argv++;
+ if(++i > max)
+ return -E2BIG;
+ }
+ }
+ return i;
+}
+
+/*
+ * 'copy_strings()' copies argument/environment strings from user
+ * memory to free pages in kernel mem. These are in a format ready
+ * to be put directly into the top of new user memory.
+ */
+int copy_strings(int argc,char __user * __user * argv, struct linux_binprm *bprm)
+{
+ struct page *kmapped_page = NULL;
+ char *kaddr = NULL;
+ int ret;
+
+ while (argc-- > 0) {
+ char __user *str;
+ int len;
+ unsigned long pos;
+
+ if (get_user(str, argv+argc) ||
+ !(len = strnlen_user(str, bprm->p))) {
+ ret = -EFAULT;
+ goto out;
+ }
+
+ if (bprm->p < len) {
+ ret = -E2BIG;
+ goto out;
+ }
+
+ bprm->p -= len;
+ /* XXX: add architecture specific overflow check here. */
+ pos = bprm->p;
+
+ while (len > 0) {
+ int i, new, err;
+ int offset, bytes_to_copy;
+ struct page *page;
+
+ offset = pos % PAGE_SIZE;
+ i = pos/PAGE_SIZE;
+ page = bprm->page[i];
+ new = 0;
+ if (!page) {
+ page = alloc_page(GFP_HIGHUSER);
+ bprm->page[i] = page;
+ if (!page) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ new = 1;
+ }
+
+ if (page != kmapped_page) {
+ if (kmapped_page)
+ kunmap(kmapped_page);
+ kmapped_page = page;
+ kaddr = kmap(kmapped_page);
+ }
+ if (new && offset)
+ memset(kaddr, 0, offset);
+ bytes_to_copy = PAGE_SIZE - offset;
+ if (bytes_to_copy > len) {
+ bytes_to_copy = len;
+ if (new)
+ memset(kaddr+offset+len, 0,
+ PAGE_SIZE-offset-len);
+ }
+ err = copy_from_user(kaddr+offset, str, bytes_to_copy);
+ if (err) {
+ ret = -EFAULT;
+ goto out;
+ }
+
+ pos += bytes_to_copy;
+ str += bytes_to_copy;
+ len -= bytes_to_copy;
+ }
+ }
+ ret = 0;
+out:
+ if (kmapped_page)
+ kunmap(kmapped_page);
+ return ret;
+}
+
+/*
+ * Like copy_strings, but get argv and its values from kernel memory.
+ */
+int copy_strings_kernel(int argc,char ** argv, struct linux_binprm *bprm)
+{
+ int r;
+ mm_segment_t oldfs = get_fs();
+ set_fs(KERNEL_DS);
+ r = copy_strings(argc, (char __user * __user *)argv, bprm);
+ set_fs(oldfs);
+ return r;
+}
+
+EXPORT_SYMBOL(copy_strings_kernel);
+
+#ifdef CONFIG_MMU
+/*
+ * This routine is used to map in a page into an address space: needed by
+ * execve() for the initial stack and environment pages.
+ *
+ * vma->vm_mm->mmap_sem is held for writing.
+ */
+void install_arg_page(struct vm_area_struct *vma,
+ struct page *page, unsigned long address)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ pgd_t * pgd;
+ pmd_t * pmd;
+ pte_t * pte;
+
+ if (unlikely(anon_vma_prepare(vma)))
+ goto out_sig;
+
+ flush_dcache_page(page);
+ pgd = pgd_offset(mm, address);
+
+ spin_lock(&mm->page_table_lock);
+ pmd = pmd_alloc(mm, pgd, address);
+ if (!pmd)
+ goto out;
+ pte = pte_alloc_map(mm, pmd, address);
+ if (!pte)
+ goto out;
+ if (!pte_none(*pte)) {
+ pte_unmap(pte);
+ goto out;
+ }
+ mm->rss++;
+ lru_cache_add_active(page);
+ set_pte(pte, pte_mkdirty(pte_mkwrite(mk_pte(
+ page, vma->vm_page_prot))));
+#ifdef CONFIG_XEN_BATCH_MODE2
+ XEN_flush_page_update_queue();
+#endif
+ page_add_anon_rmap(page, vma, address);
+ pte_unmap(pte);
+ spin_unlock(&mm->page_table_lock);
+
+ /* no need for flush_tlb */
+ return;
+out:
+ spin_unlock(&mm->page_table_lock);
+out_sig:
+ __free_page(page);
+ force_sig(SIGKILL, current);
+}
+
+int setup_arg_pages(struct linux_binprm *bprm, int executable_stack)
+{
+ unsigned long stack_base;
+ struct vm_area_struct *mpnt;
+ struct mm_struct *mm = current->mm;
+ int i, ret;
+ long arg_size;
+
+#ifdef CONFIG_STACK_GROWSUP
+ /* Move the argument and environment strings to the bottom of the
+ * stack space.
+ */
+ int offset, j;
+ char *to, *from;
+
+ /* Start by shifting all the pages down */
+ i = 0;
+ for (j = 0; j < MAX_ARG_PAGES; j++) {
+ struct page *page = bprm->page[j];
+ if (!page)
+ continue;
+ bprm->page[i++] = page;
+ }
+
+ /* Now move them within their pages */
+ offset = bprm->p % PAGE_SIZE;
+ to = kmap(bprm->page[0]);
+ for (j = 1; j < i; j++) {
+ memmove(to, to + offset, PAGE_SIZE - offset);
+ from = kmap(bprm->page[j]);
+ memcpy(to + PAGE_SIZE - offset, from, offset);
+ kunmap(bprm->page[j - 1]);
+ to = from;
+ }
+ memmove(to, to + offset, PAGE_SIZE - offset);
+ kunmap(bprm->page[j - 1]);
+
+ /* Adjust bprm->p to point to the end of the strings. */
+ bprm->p = PAGE_SIZE * i - offset;
+
+ /* Limit stack size to 1GB */
+ stack_base = current->signal->rlim[RLIMIT_STACK].rlim_max;
+ if (stack_base > (1 << 30))
+ stack_base = 1 << 30;
+ stack_base = PAGE_ALIGN(STACK_TOP - stack_base);
+
+ mm->arg_start = stack_base;
+ arg_size = i << PAGE_SHIFT;
+
+ /* zero pages that were copied above */
+ while (i < MAX_ARG_PAGES)
+ bprm->page[i++] = NULL;
+#else
+ stack_base = STACK_TOP - MAX_ARG_PAGES * PAGE_SIZE;
+ mm->arg_start = bprm->p + stack_base;
+ arg_size = STACK_TOP - (PAGE_MASK & (unsigned long) mm->arg_start);
+#endif
+
+ bprm->p += stack_base;
+ if (bprm->loader)
+ bprm->loader += stack_base;
+ bprm->exec += stack_base;
+
+ mpnt = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
+ if (!mpnt)
+ return -ENOMEM;
+
+ if (security_vm_enough_memory(arg_size >> PAGE_SHIFT)) {
+ kmem_cache_free(vm_area_cachep, mpnt);
+ return -ENOMEM;
+ }
+
+ memset(mpnt, 0, sizeof(*mpnt));
+
+ down_write(&mm->mmap_sem);
+ {
+ mpnt->vm_mm = mm;
+#ifdef CONFIG_STACK_GROWSUP
+ mpnt->vm_start = stack_base;
+ mpnt->vm_end = PAGE_MASK &
+ (PAGE_SIZE - 1 + (unsigned long) bprm->p);
+#else
+ mpnt->vm_start = PAGE_MASK & (unsigned long) bprm->p;
+ mpnt->vm_end = STACK_TOP;
+#endif
+ /* Adjust stack execute permissions; explicitly enable
+ * for EXSTACK_ENABLE_X, disable for EXSTACK_DISABLE_X
+ * and leave alone (arch default) otherwise. */
+ if (unlikely(executable_stack == EXSTACK_ENABLE_X))
+ mpnt->vm_flags = VM_STACK_FLAGS | VM_EXEC;
+ else if (executable_stack == EXSTACK_DISABLE_X)
+ mpnt->vm_flags = VM_STACK_FLAGS & ~VM_EXEC;
+ else
+ mpnt->vm_flags = VM_STACK_FLAGS;
+ mpnt->vm_flags |= mm->def_flags;
+ mpnt->vm_page_prot = protection_map[mpnt->vm_flags & 0x7];
+ if ((ret = insert_vm_struct(mm, mpnt))) {
+ up_write(&mm->mmap_sem);
+ kmem_cache_free(vm_area_cachep, mpnt);
+ return ret;
+ }
+ mm->stack_vm = mm->total_vm = vma_pages(mpnt);
+ }
+
+ for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
+ struct page *page = bprm->page[i];
+ if (page) {
+ bprm->page[i] = NULL;
+ install_arg_page(mpnt, page, stack_base);
+ }
+ stack_base += PAGE_SIZE;
+ }
+ up_write(&mm->mmap_sem);
+
+ return 0;
+}
+
+EXPORT_SYMBOL(setup_arg_pages);
+
+#define free_arg_pages(bprm) do { } while (0)
+
+#else
+
+static inline void free_arg_pages(struct linux_binprm *bprm)
+{
+ int i;
+
+ for (i = 0; i < MAX_ARG_PAGES; i++) {
+ if (bprm->page[i])
+ __free_page(bprm->page[i]);
+ bprm->page[i] = NULL;
+ }
+}
+
+#endif /* CONFIG_MMU */
+
+struct file *open_exec(const char *name)
+{
+ struct nameidata nd;
+ int err;
+ struct file *file;
+
+ nd.intent.open.flags = FMODE_READ;
+ err = path_lookup(name, LOOKUP_FOLLOW|LOOKUP_OPEN, &nd);
+ file = ERR_PTR(err);
+
+ if (!err) {
+ struct inode *inode = nd.dentry->d_inode;
+ file = ERR_PTR(-EACCES);
+ if (!(nd.mnt->mnt_flags & MNT_NOEXEC) &&
+ S_ISREG(inode->i_mode)) {
+ int err = permission(inode, MAY_EXEC, &nd);
+ if (!err && !(inode->i_mode & 0111))
+ err = -EACCES;
+ file = ERR_PTR(err);
+ if (!err) {
+ file = dentry_open(nd.dentry, nd.mnt, O_RDONLY);
+ if (!IS_ERR(file)) {
+ err = deny_write_access(file);
+ if (err) {
+ fput(file);
+ file = ERR_PTR(err);
+ }
+ }
+out:
+ return file;
+ }
+ }
+ path_release(&nd);
+ }
+ goto out;
+}
+
+EXPORT_SYMBOL(open_exec);
+
+int kernel_read(struct file *file, unsigned long offset,
+ char *addr, unsigned long count)
+{
+ mm_segment_t old_fs;
+ loff_t pos = offset;
+ int result;
+
+ old_fs = get_fs();
+ set_fs(get_ds());
+ /* The cast to a user pointer is valid due to the set_fs() */
+ result = vfs_read(file, (void __user *)addr, count, &pos);
+ set_fs(old_fs);
+ return result;
+}
+
+EXPORT_SYMBOL(kernel_read);
+
+static int exec_mmap(struct mm_struct *mm)
+{
+ struct task_struct *tsk;
+ struct mm_struct * old_mm, *active_mm;
+
+ /* Notify parent that we're no longer interested in the old VM */
+ tsk = current;
+ old_mm = current->mm;
+ mm_release(tsk, old_mm);
+
+ task_lock(tsk);
+ active_mm = tsk->active_mm;
+ tsk->mm = mm;
+ tsk->active_mm = mm;
+ activate_mm(active_mm, mm);
+ task_unlock(tsk);
+ arch_pick_mmap_layout(mm);
+ if (old_mm) {
+ if (active_mm != old_mm) BUG();
+ mmput(old_mm);
+ return 0;
+ }
+ mmdrop(active_mm);
+ return 0;
+}
+
+/*
+ * This function makes sure the current process has its own signal table,
+ * so that flush_signal_handlers can later reset the handlers without
+ * disturbing other processes. (Other processes might share the signal
+ * table via the CLONE_SIGHAND option to clone().)
+ */
+static inline int de_thread(struct task_struct *tsk)
+{
+ struct signal_struct *sig = tsk->signal;
+ struct sighand_struct *newsighand, *oldsighand = tsk->sighand;
+ spinlock_t *lock = &oldsighand->siglock;
+ int count;
+
+ /*
+ * If we don't share sighandlers, then we aren't sharing anything
+ * and we can just re-use it all.
+ */
+ if (atomic_read(&oldsighand->count) <= 1) {
+ BUG_ON(atomic_read(&sig->count) != 1);
+ exit_itimers(sig);
+ return 0;
+ }
+
+ newsighand = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
+ if (!newsighand)
+ return -ENOMEM;
+
+ if (thread_group_empty(current))
+ goto no_thread_group;
+
+ /*
+ * Kill all other threads in the thread group.
+ * We must hold tasklist_lock to call zap_other_threads.
+ */
+ read_lock(&tasklist_lock);
+ spin_lock_irq(lock);
+ if (sig->group_exit) {
+ /*
+ * Another group action in progress, just
+ * return so that the signal is processed.
+ */
+ spin_unlock_irq(lock);
+ read_unlock(&tasklist_lock);
+ kmem_cache_free(sighand_cachep, newsighand);
+ return -EAGAIN;
+ }
+ sig->group_exit = 1;
+ zap_other_threads(current);
+ read_unlock(&tasklist_lock);
+
+ /*
+ * Account for the thread group leader hanging around:
+ */
+ count = 2;
+ if (current->pid == current->tgid)
+ count = 1;
+ while (atomic_read(&sig->count) > count) {
+ sig->group_exit_task = current;
+ sig->notify_count = count;
+ __set_current_state(TASK_UNINTERRUPTIBLE);
+ spin_unlock_irq(lock);
+ schedule();
+ spin_lock_irq(lock);
+ }
+ sig->group_exit_task = NULL;
+ sig->notify_count = 0;
+ spin_unlock_irq(lock);
+
+ /*
+ * At this point all other threads have exited, all we have to
+ * do is to wait for the thread group leader to become inactive,
+ * and to assume its PID:
+ */
+ if (current->pid != current->tgid) {
+ struct task_struct *leader = current->group_leader, *parent;
+ struct dentry *proc_dentry1, *proc_dentry2;
+ unsigned long exit_state, ptrace;
+
+ /*
+ * Wait for the thread group leader to be a zombie.
+ * It should already be zombie at this point, most
+ * of the time.
+ */
+ while (leader->exit_state != EXIT_ZOMBIE)
+ yield();
+
+ spin_lock(&leader->proc_lock);
+ spin_lock(¤t->proc_lock);
+ proc_dentry1 = proc_pid_unhash(current);
+ proc_dentry2 = proc_pid_unhash(leader);
+ write_lock_irq(&tasklist_lock);
+
+ if (leader->tgid != current->tgid)
+ BUG();
+ if (current->pid == current->tgid)
+ BUG();
+ /*
+ * An exec() starts a new thread group with the
+ * TGID of the previous thread group. Rehash the
+ * two threads with a switched PID, and release
+ * the former thread group leader:
+ */
+ ptrace = leader->ptrace;
+ parent = leader->parent;
+
+ ptrace_unlink(current);
+ ptrace_unlink(leader);
+ remove_parent(current);
+ remove_parent(leader);
+
+ switch_exec_pids(leader, current);
+
+ current->parent = current->real_parent = leader->real_parent;
+ leader->parent = leader->real_parent = child_reaper;
+ current->group_leader = current;
+ leader->group_leader = leader;
+
+ add_parent(current, current->parent);
+ add_parent(leader, leader->parent);
+ if (ptrace) {
+ current->ptrace = ptrace;
+ __ptrace_link(current, parent);
+ }
+
+ list_del(¤t->tasks);
+ list_add_tail(¤t->tasks, &init_task.tasks);
+ current->exit_signal = SIGCHLD;
+ exit_state = leader->exit_state;
+
+ write_unlock_irq(&tasklist_lock);
+ spin_unlock(&leader->proc_lock);
+ spin_unlock(¤t->proc_lock);
+ proc_pid_flush(proc_dentry1);
+ proc_pid_flush(proc_dentry2);
+
+ if (exit_state != EXIT_ZOMBIE)
+ BUG();
+ release_task(leader);
+ }
+
+ /*
+ * Now there are really no other threads at all,
+ * so it's safe to stop telling them to kill themselves.
+ */
+ sig->group_exit = 0;
+
+no_thread_group:
+ BUG_ON(atomic_read(&sig->count) != 1);
+ exit_itimers(sig);
+
+ if (atomic_read(&oldsighand->count) == 1) {
+ /*
+ * Now that we nuked the rest of the thread group,
+ * it turns out we are not sharing sighand any more either.
+ * So we can just keep it.
+ */
+ kmem_cache_free(sighand_cachep, newsighand);
+ } else {
+ /*
+ * Move our state over to newsighand and switch it in.
+ */
+ spin_lock_init(&newsighand->siglock);
+ atomic_set(&newsighand->count, 1);
+ memcpy(newsighand->action, oldsighand->action,
+ sizeof(newsighand->action));
+
+ write_lock_irq(&tasklist_lock);
+ spin_lock(&oldsighand->siglock);
+ spin_lock(&newsighand->siglock);
+
+ current->sighand = newsighand;
+ recalc_sigpending();
+
+ spin_unlock(&newsighand->siglock);
+ spin_unlock(&oldsighand->siglock);
+ write_unlock_irq(&tasklist_lock);
+
+ if (atomic_dec_and_test(&oldsighand->count))
+ kmem_cache_free(sighand_cachep, oldsighand);
+ }
+
+ if (!thread_group_empty(current))
+ BUG();
+ if (current->tgid != current->pid)
+ BUG();
+ return 0;
+}
+
+/*
+ * These functions flushes out all traces of the currently running executable
+ * so that a new one can be started
+ */
+
+static inline void flush_old_files(struct files_struct * files)
+{
+ long j = -1;
+
+ spin_lock(&files->file_lock);
+ for (;;) {
+ unsigned long set, i;
+
+ j++;
+ i = j * __NFDBITS;
+ if (i >= files->max_fds || i >= files->max_fdset)
+ break;
+ set = files->close_on_exec->fds_bits[j];
+ if (!set)
+ continue;
+ files->close_on_exec->fds_bits[j] = 0;
+ spin_unlock(&files->file_lock);
+ for ( ; set ; i++,set >>= 1) {
+ if (set & 1) {
+ sys_close(i);
+ }
+ }
+ spin_lock(&files->file_lock);
+
+ }
+ spin_unlock(&files->file_lock);
+}
+
+void get_task_comm(char *buf, struct task_struct *tsk)
+{
+ /* buf must be at least sizeof(tsk->comm) in size */
+ task_lock(tsk);
+ memcpy(buf, tsk->comm, sizeof(tsk->comm));
+ task_unlock(tsk);
+}
+
+void set_task_comm(struct task_struct *tsk, char *buf)
+{
+ task_lock(tsk);
+ strlcpy(tsk->comm, buf, sizeof(tsk->comm));
+ task_unlock(tsk);
+}
+
+int flush_old_exec(struct linux_binprm * bprm)
+{
+ char * name;
+ int i, ch, retval;
+ struct files_struct *files;
+ char tcomm[sizeof(current->comm)];
+
+ /*
+ * Make sure we have a private signal table and that
+ * we are unassociated from the previous thread group.
+ */
+ retval = de_thread(current);
+ if (retval)
+ goto out;
+
+ /*
+ * Make sure we have private file handles. Ask the
+ * fork helper to do the work for us and the exit
+ * helper to do the cleanup of the old one.
+ */
+ files = current->files; /* refcounted so safe to hold */
+ retval = unshare_files();
+ if (retval)
+ goto out;
+ /*
+ * Release all of the old mmap stuff
+ */
+ retval = exec_mmap(bprm->mm);
+ if (retval)
+ goto mmap_failed;
+
+ bprm->mm = NULL; /* We're using it now */
+
+ /* This is the point of no return */
+ steal_locks(files);
+ put_files_struct(files);
+
+ current->sas_ss_sp = current->sas_ss_size = 0;
+
+ if (current->euid == current->uid && current->egid == current->gid)
+ current->mm->dumpable = 1;
+ name = bprm->filename;
+ for (i=0; (ch = *(name++)) != '\0';) {
+ if (ch == '/')
+ i = 0;
+ else
+ if (i < (sizeof(tcomm) - 1))
+ tcomm[i++] = ch;
+ }
+ tcomm[i] = '\0';
+ set_task_comm(current, tcomm);
+
+ flush_thread();
+
+ if (bprm->e_uid != current->euid || bprm->e_gid != current->egid ||
+ permission(bprm->file->f_dentry->d_inode,MAY_READ, NULL) ||
+ (bprm->interp_flags & BINPRM_FLAGS_ENFORCE_NONDUMP)) {
+ suid_keys(current);
+ current->mm->dumpable = 0;
+ }
+
+ /* An exec changes our domain. We are no longer part of the thread
+ group */
+
+ current->self_exec_id++;
+
+ flush_signal_handlers(current, 0);
+ flush_old_files(current->files);
+
+ return 0;
+
+mmap_failed:
+ put_files_struct(current->files);
+ current->files = files;
+out:
+ return retval;
+}
+
+EXPORT_SYMBOL(flush_old_exec);
+
+/*
+ * Fill the binprm structure from the inode.
+ * Check permissions, then read the first 128 (BINPRM_BUF_SIZE) bytes
+ */
+int prepare_binprm(struct linux_binprm *bprm)
+{
+ int mode;
+ struct inode * inode = bprm->file->f_dentry->d_inode;
+ int retval;
+
+ mode = inode->i_mode;
+ /*
+ * Check execute perms again - if the caller has CAP_DAC_OVERRIDE,
+ * generic_permission lets a non-executable through
+ */
+ if (!(mode & 0111)) /* with at least _one_ execute bit set */
+ return -EACCES;
+ if (bprm->file->f_op == NULL)
+ return -EACCES;
+
+ bprm->e_uid = current->euid;
+ bprm->e_gid = current->egid;
+
+ if(!(bprm->file->f_vfsmnt->mnt_flags & MNT_NOSUID)) {
+ /* Set-uid? */
+ if (mode & S_ISUID) {
+ current->personality &= ~PER_CLEAR_ON_SETID;
+ bprm->e_uid = inode->i_uid;
+ }
+
+ /* Set-gid? */
+ /*
+ * If setgid is set but no group execute bit then this
+ * is a candidate for mandatory locking, not a setgid
+ * executable.
+ */
+ if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) {
+ current->personality &= ~PER_CLEAR_ON_SETID;
+ bprm->e_gid = inode->i_gid;
+ }
+ }
+
+ /* fill in binprm security blob */
+ retval = security_bprm_set(bprm);
+ if (retval)
+ return retval;
+
+ memset(bprm->buf,0,BINPRM_BUF_SIZE);
+ return kernel_read(bprm->file,0,bprm->buf,BINPRM_BUF_SIZE);
+}
+
+EXPORT_SYMBOL(prepare_binprm);
+
+static inline int unsafe_exec(struct task_struct *p)
+{
+ int unsafe = 0;
+ if (p->ptrace & PT_PTRACED) {
+ if (p->ptrace & PT_PTRACE_CAP)
+ unsafe |= LSM_UNSAFE_PTRACE_CAP;
+ else
+ unsafe |= LSM_UNSAFE_PTRACE;
+ }
+ if (atomic_read(&p->fs->count) > 1 ||
+ atomic_read(&p->files->count) > 1 ||
+ atomic_read(&p->sighand->count) > 1)
+ unsafe |= LSM_UNSAFE_SHARE;
+
+ return unsafe;
+}
+
+void compute_creds(struct linux_binprm *bprm)
+{
+ int unsafe;
+
+ if (bprm->e_uid != current->uid)
+ suid_keys(current);
+ exec_keys(current);
+
+ task_lock(current);
+ unsafe = unsafe_exec(current);
+ security_bprm_apply_creds(bprm, unsafe);
+ task_unlock(current);
+}
+
+EXPORT_SYMBOL(compute_creds);
+
+void remove_arg_zero(struct linux_binprm *bprm)
+{
+ if (bprm->argc) {
+ unsigned long offset;
+ char * kaddr;
+ struct page *page;
+
+ offset = bprm->p % PAGE_SIZE;
+ goto inside;
+
+ while (bprm->p++, *(kaddr+offset++)) {
+ if (offset != PAGE_SIZE)
+ continue;
+ offset = 0;
+ kunmap_atomic(kaddr, KM_USER0);
+inside:
+ page = bprm->page[bprm->p/PAGE_SIZE];
+ kaddr = kmap_atomic(page, KM_USER0);
+ }
+ kunmap_atomic(kaddr, KM_USER0);
+ bprm->argc--;
+ }
+}
+
+EXPORT_SYMBOL(remove_arg_zero);
+
+/*
+ * cycle the list of binary formats handler, until one recognizes the image
+ */
+int search_binary_handler(struct linux_binprm *bprm,struct pt_regs *regs)
+{
+ int try,retval;
+ struct linux_binfmt *fmt;
+#ifdef __alpha__
+ /* handle /sbin/loader.. */
+ {
+ struct exec * eh = (struct exec *) bprm->buf;
+
+ if (!bprm->loader && eh->fh.f_magic == 0x183 &&
+ (eh->fh.f_flags & 0x3000) == 0x3000)
+ {
+ struct file * file;
+ unsigned long loader;
+
+ allow_write_access(bprm->file);
+ fput(bprm->file);
+ bprm->file = NULL;
+
+ loader = PAGE_SIZE*MAX_ARG_PAGES-sizeof(void *);
+
+ file = open_exec("/sbin/loader");
+ retval = PTR_ERR(file);
+ if (IS_ERR(file))
+ return retval;
+
+ /* Remember if the application is TASO. */
+ bprm->sh_bang = eh->ah.entry < 0x100000000UL;
+
+ bprm->file = file;
+ bprm->loader = loader;
+ retval = prepare_binprm(bprm);
+ if (retval<0)
+ return retval;
+ /* should call search_binary_handler recursively here,
+ but it does not matter */
+ }
+ }
+#endif
+ retval = security_bprm_check(bprm);
+ if (retval)
+ return retval;
+
+ /* kernel module loader fixup */
+ /* so we don't try to load run modprobe in kernel space. */
+ set_fs(USER_DS);
+ retval = -ENOENT;
+ for (try=0; try<2; try++) {
+ read_lock(&binfmt_lock);
+ for (fmt = formats ; fmt ; fmt = fmt->next) {
+ int (*fn)(struct linux_binprm *, struct pt_regs *) = fmt->load_binary;
+ if (!fn)
+ continue;
+ if (!try_module_get(fmt->module))
+ continue;
+ read_unlock(&binfmt_lock);
+ retval = fn(bprm, regs);
+ if (retval >= 0) {
+ put_binfmt(fmt);
+ allow_write_access(bprm->file);
+ if (bprm->file)
+ fput(bprm->file);
+ bprm->file = NULL;
+ current->did_exec = 1;
+ return retval;
+ }
+ read_lock(&binfmt_lock);
+ put_binfmt(fmt);
+ if (retval != -ENOEXEC || bprm->mm == NULL)
+ break;
+ if (!bprm->file) {
+ read_unlock(&binfmt_lock);
+ return retval;
+ }
+ }
+ read_unlock(&binfmt_lock);
+ if (retval != -ENOEXEC || bprm->mm == NULL) {
+ break;
+#ifdef CONFIG_KMOD
+ }else{
+#define printable(c) (((c)=='\t') || ((c)=='\n') || (0x20<=(c) && (c)<=0x7e))
+ if (printable(bprm->buf[0]) &&
+ printable(bprm->buf[1]) &&
+ printable(bprm->buf[2]) &&
+ printable(bprm->buf[3]))
+ break; /* -ENOEXEC */
+ request_module("binfmt-%04x", *(unsigned short *)(&bprm->buf[2]));
+#endif
+ }
+ }
+ return retval;
+}
+
+EXPORT_SYMBOL(search_binary_handler);
+
+/*
+ * sys_execve() executes a new program.
+ */
+int do_execve(char * filename,
+ char __user *__user *argv,
+ char __user *__user *envp,
+ struct pt_regs * regs)
+{
+ struct linux_binprm *bprm;
+ struct file *file;
+ int retval;
+ int i;
+
+ retval = -ENOMEM;
+ bprm = kmalloc(sizeof(*bprm), GFP_KERNEL);
+ if (!bprm)
+ goto out_ret;
+ memset(bprm, 0, sizeof(*bprm));
+
+ file = open_exec(filename);
+ retval = PTR_ERR(file);
+ if (IS_ERR(file))
+ goto out_kfree;
+
+ sched_exec();
+
+ bprm->p = PAGE_SIZE*MAX_ARG_PAGES-sizeof(void *);
+
+ bprm->file = file;
+ bprm->filename = filename;
+ bprm->interp = filename;
+ bprm->mm = mm_alloc();
+ retval = -ENOMEM;
+ if (!bprm->mm)
+ goto out_file;
+
+ retval = init_new_context(current, bprm->mm);
+ if (retval < 0)
+ goto out_mm;
+
+ bprm->argc = count(argv, bprm->p / sizeof(void *));
+ if ((retval = bprm->argc) < 0)
+ goto out_mm;
+
+ bprm->envc = count(envp, bprm->p / sizeof(void *));
+ if ((retval = bprm->envc) < 0)
+ goto out_mm;
+
+ retval = security_bprm_alloc(bprm);
+ if (retval)
+ goto out;
+
+ retval = prepare_binprm(bprm);
+ if (retval < 0)
+ goto out;
+
+ retval = copy_strings_kernel(1, &bprm->filename, bprm);
+ if (retval < 0)
+ goto out;
+
+ bprm->exec = bprm->p;
+ retval = copy_strings(bprm->envc, envp, bprm);
+ if (retval < 0)
+ goto out;
+
+ retval = copy_strings(bprm->argc, argv, bprm);
+ if (retval < 0)
+ goto out;
+
+ retval = search_binary_handler(bprm,regs);
+ if (retval >= 0) {
+ free_arg_pages(bprm);
+
+ /* execve success */
+ security_bprm_free(bprm);
+ kfree(bprm);
+ return retval;
+ }
+
+out:
+ /* Something went wrong, return the inode and free the argument pages*/
+ for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
+ struct page * page = bprm->page[i];
+ if (page)
+ __free_page(page);
+ }
+
+ if (bprm->security)
+ security_bprm_free(bprm);
+
+out_mm:
+ if (bprm->mm)
+ mmdrop(bprm->mm);
+
+out_file:
+ if (bprm->file) {
+ allow_write_access(bprm->file);
+ fput(bprm->file);
+ }
+
+out_kfree:
+ kfree(bprm);
+
+out_ret:
+ return retval;
+}
+
+int set_binfmt(struct linux_binfmt *new)
+{
+ struct linux_binfmt *old = current->binfmt;
+
+ if (new) {
+ if (!try_module_get(new->module))
+ return -1;
+ }
+ current->binfmt = new;
+ if (old)
+ module_put(old->module);
+ return 0;
+}
+
+EXPORT_SYMBOL(set_binfmt);
+
+#define CORENAME_MAX_SIZE 64
+
+/* format_corename will inspect the pattern parameter, and output a
+ * name into corename, which must have space for at least
+ * CORENAME_MAX_SIZE bytes plus one byte for the zero terminator.
+ */
+static void format_corename(char *corename, const char *pattern, long signr)
+{
+ const char *pat_ptr = pattern;
+ char *out_ptr = corename;
+ char *const out_end = corename + CORENAME_MAX_SIZE;
+ int rc;
+ int pid_in_pattern = 0;
+
+ /* Repeat as long as we have more pattern to process and more output
+ space */
+ while (*pat_ptr) {
+ if (*pat_ptr != '%') {
+ if (out_ptr == out_end)
+ goto out;
+ *out_ptr++ = *pat_ptr++;
+ } else {
+ switch (*++pat_ptr) {
+ case 0:
+ goto out;
+ /* Double percent, output one percent */
+ case '%':
+ if (out_ptr == out_end)
+ goto out;
+ *out_ptr++ = '%';
+ break;
+ /* pid */
+ case 'p':
+ pid_in_pattern = 1;
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%d", current->tgid);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ /* uid */
+ case 'u':
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%d", current->uid);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ /* gid */
+ case 'g':
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%d", current->gid);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ /* signal that caused the coredump */
+ case 's':
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%ld", signr);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ /* UNIX time of coredump */
+ case 't': {
+ struct timeval tv;
+ do_gettimeofday(&tv);
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%lu", tv.tv_sec);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ }
+ /* hostname */
+ case 'h':
+ down_read(&uts_sem);
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%s", system_utsname.nodename);
+ up_read(&uts_sem);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ /* executable */
+ case 'e':
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ "%s", current->comm);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ break;
+ default:
+ break;
+ }
+ ++pat_ptr;
+ }
+ }
+ /* Backward compatibility with core_uses_pid:
+ *
+ * If core_pattern does not include a %p (as is the default)
+ * and core_uses_pid is set, then .%pid will be appended to
+ * the filename */
+ if (!pid_in_pattern
+ && (core_uses_pid || atomic_read(¤t->mm->mm_users) != 1)) {
+ rc = snprintf(out_ptr, out_end - out_ptr,
+ ".%d", current->tgid);
+ if (rc > out_end - out_ptr)
+ goto out;
+ out_ptr += rc;
+ }
+ out:
+ *out_ptr = 0;
+}
+
+static void zap_threads (struct mm_struct *mm)
+{
+ struct task_struct *g, *p;
+ struct task_struct *tsk = current;
+ struct completion *vfork_done = tsk->vfork_done;
+
+ /*
+ * Make sure nobody is waiting for us to release the VM,
+ * otherwise we can deadlock when we wait on each other
+ */
+ if (vfork_done) {
+ tsk->vfork_done = NULL;
+ complete(vfork_done);
+ }
+
+ read_lock(&tasklist_lock);
+ do_each_thread(g,p)
+ if (mm == p->mm && p != tsk) {
+ force_sig_specific(SIGKILL, p);
+ mm->core_waiters++;
+ }
+ while_each_thread(g,p);
+
+ read_unlock(&tasklist_lock);
+}
+
+static void coredump_wait(struct mm_struct *mm)
+{
+ DECLARE_COMPLETION(startup_done);
+
+ mm->core_waiters++; /* let other threads block */
+ mm->core_startup_done = &startup_done;
+
+ /* give other threads a chance to run: */
+ yield();
+
+ zap_threads(mm);
+ if (--mm->core_waiters) {
+ up_write(&mm->mmap_sem);
+ wait_for_completion(&startup_done);
+ } else
+ up_write(&mm->mmap_sem);
+ BUG_ON(mm->core_waiters);
+}
+
+int do_coredump(long signr, int exit_code, struct pt_regs * regs)
+{
+ char corename[CORENAME_MAX_SIZE + 1];
+ struct mm_struct *mm = current->mm;
+ struct linux_binfmt * binfmt;
+ struct inode * inode;
+ struct file * file;
+ int retval = 0;
+
+ binfmt = current->binfmt;
+ if (!binfmt || !binfmt->core_dump)
+ goto fail;
+ down_write(&mm->mmap_sem);
+ if (!mm->dumpable) {
+ up_write(&mm->mmap_sem);
+ goto fail;
+ }
+ mm->dumpable = 0;
+ init_completion(&mm->core_done);
+ current->signal->group_exit = 1;
+ current->signal->group_exit_code = exit_code;
+ coredump_wait(mm);
+
+ if (current->signal->rlim[RLIMIT_CORE].rlim_cur < binfmt->min_coredump)
+ goto fail_unlock;
+
+ /*
+ * lock_kernel() because format_corename() is controlled by sysctl, which
+ * uses lock_kernel()
+ */
+ lock_kernel();
+ format_corename(corename, core_pattern, signr);
+ unlock_kernel();
+ file = filp_open(corename, O_CREAT | 2 | O_NOFOLLOW | O_LARGEFILE, 0600);
+ if (IS_ERR(file))
+ goto fail_unlock;
+ inode = file->f_dentry->d_inode;
+ if (inode->i_nlink > 1)
+ goto close_fail; /* multiple links - don't dump */
+ if (d_unhashed(file->f_dentry))
+ goto close_fail;
+
+ if (!S_ISREG(inode->i_mode))
+ goto close_fail;
+ if (!file->f_op)
+ goto close_fail;
+ if (!file->f_op->write)
+ goto close_fail;
+ if (do_truncate(file->f_dentry, 0) != 0)
+ goto close_fail;
+
+ retval = binfmt->core_dump(signr, regs, file);
+
+ if (retval)
+ current->signal->group_exit_code |= 0x80;
+close_fail:
+ filp_close(file, NULL);
+fail_unlock:
+ complete_all(&mm->core_done);
+fail:
+ return retval;
+}
#if defined(CONFIG_XEN_DEBUG_NO_MMU_BATCHING)
#define set_pte(pteptr, pteval)\
set_pte_batched(pteptr, pteval)
-#elif defined(CONFIG_XEN_BATCH_MODE)
+
+#elif defined(CONFIG_XEN_BATCH_MODE1)
#define set_pte(pteptr, pteval)({\
set_pte_batched(pteptr, pteval);\
_flush_page_update_queue();})
+
+#elif defined(CONFIG_XEN_BATCH_MODE2)
+#define set_pte(pteptr, pteval)\
+ set_pte_batched(pteptr, pteval)
+
#else
#define set_pte(pteptr, pteval) (*(pteptr) = pteval)
#endif
--- /dev/null
+/*
+ * High memory handling common code and variables.
+ *
+ * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
+ * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
+ *
+ *
+ * Redesigned the x86 32-bit VM architecture to deal with
+ * 64-bit physical space. With current x86 CPUs this
+ * means up to 64 Gigabytes physical RAM.
+ *
+ * Rewrote high memory support to move the page cache into
+ * high memory. Implemented permanent (schedulable) kmaps
+ * based on Linus' idea.
+ *
+ * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/swap.h>
+#include <linux/bio.h>
+#include <linux/pagemap.h>
+#include <linux/mempool.h>
+#include <linux/blkdev.h>
+#include <linux/init.h>
+#include <linux/hash.h>
+#include <linux/highmem.h>
+#include <asm/tlbflush.h>
+
+static mempool_t *page_pool, *isa_page_pool;
+
+static void *page_pool_alloc(int gfp_mask, void *data)
+{
+ int gfp = gfp_mask | (int) (long) data;
+
+ return alloc_page(gfp);
+}
+
+static void page_pool_free(void *page, void *data)
+{
+ __free_page(page);
+}
+
+/*
+ * Virtual_count is not a pure "count".
+ * 0 means that it is not mapped, and has not been mapped
+ * since a TLB flush - it is usable.
+ * 1 means that there are no users, but it has been mapped
+ * since the last TLB flush - so we can't use it.
+ * n means that there are (n-1) current users of it.
+ */
+#ifdef CONFIG_HIGHMEM
+static int pkmap_count[LAST_PKMAP];
+static unsigned int last_pkmap_nr;
+static spinlock_t kmap_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+
+pte_t * pkmap_page_table;
+
+static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
+
+static void flush_all_zero_pkmaps(void)
+{
+ int i;
+
+ flush_cache_kmaps();
+
+ for (i = 0; i < LAST_PKMAP; i++) {
+ struct page *page;
+
+ /*
+ * zero means we don't have anything to do,
+ * >1 means that it is still in use. Only
+ * a count of 1 means that it is free but
+ * needs to be unmapped
+ */
+ if (pkmap_count[i] != 1)
+ continue;
+ pkmap_count[i] = 0;
+
+ /* sanity check */
+ if (pte_none(pkmap_page_table[i]))
+ BUG();
+
+ /*
+ * Don't need an atomic fetch-and-clear op here;
+ * no-one has the page mapped, and cannot get at
+ * its virtual address (and hence PTE) without first
+ * getting the kmap_lock (which is held here).
+ * So no dangers, even with speculative execution.
+ */
+ page = pte_page(pkmap_page_table[i]);
+ pte_clear(&pkmap_page_table[i]);
+
+ set_page_address(page, NULL);
+ }
+ flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
+}
+
+static inline unsigned long map_new_virtual(struct page *page)
+{
+ unsigned long vaddr;
+ int count;
+
+start:
+ count = LAST_PKMAP;
+ /* Find an empty entry */
+ for (;;) {
+ last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
+ if (!last_pkmap_nr) {
+ flush_all_zero_pkmaps();
+ count = LAST_PKMAP;
+ }
+ if (!pkmap_count[last_pkmap_nr])
+ break; /* Found a usable entry */
+ if (--count)
+ continue;
+
+ /*
+ * Sleep for somebody else to unmap their entries
+ */
+ {
+ DECLARE_WAITQUEUE(wait, current);
+
+ __set_current_state(TASK_UNINTERRUPTIBLE);
+ add_wait_queue(&pkmap_map_wait, &wait);
+ spin_unlock(&kmap_lock);
+ schedule();
+ remove_wait_queue(&pkmap_map_wait, &wait);
+ spin_lock(&kmap_lock);
+
+ /* Somebody else might have mapped it while we slept */
+ if (page_address(page))
+ return (unsigned long)page_address(page);
+
+ /* Re-start */
+ goto start;
+ }
+ }
+ vaddr = PKMAP_ADDR(last_pkmap_nr);
+ set_pte(&(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
+#ifdef CONFIG_XEN_BATCH_MODE2
+ XEN_flush_page_update_queue();
+#endif
+ pkmap_count[last_pkmap_nr] = 1;
+ set_page_address(page, (void *)vaddr);
+
+ return vaddr;
+}
+
+void fastcall *kmap_high(struct page *page)
+{
+ unsigned long vaddr;
+
+ /*
+ * For highmem pages, we can't trust "virtual" until
+ * after we have the lock.
+ *
+ * We cannot call this from interrupts, as it may block
+ */
+ spin_lock(&kmap_lock);
+ vaddr = (unsigned long)page_address(page);
+ if (!vaddr)
+ vaddr = map_new_virtual(page);
+ pkmap_count[PKMAP_NR(vaddr)]++;
+ if (pkmap_count[PKMAP_NR(vaddr)] < 2)
+ BUG();
+ spin_unlock(&kmap_lock);
+ return (void*) vaddr;
+}
+
+EXPORT_SYMBOL(kmap_high);
+
+void fastcall kunmap_high(struct page *page)
+{
+ unsigned long vaddr;
+ unsigned long nr;
+ int need_wakeup;
+
+ spin_lock(&kmap_lock);
+ vaddr = (unsigned long)page_address(page);
+ if (!vaddr)
+ BUG();
+ nr = PKMAP_NR(vaddr);
+
+ /*
+ * A count must never go down to zero
+ * without a TLB flush!
+ */
+ need_wakeup = 0;
+ switch (--pkmap_count[nr]) {
+ case 0:
+ BUG();
+ case 1:
+ /*
+ * Avoid an unnecessary wake_up() function call.
+ * The common case is pkmap_count[] == 1, but
+ * no waiters.
+ * The tasks queued in the wait-queue are guarded
+ * by both the lock in the wait-queue-head and by
+ * the kmap_lock. As the kmap_lock is held here,
+ * no need for the wait-queue-head's lock. Simply
+ * test if the queue is empty.
+ */
+ need_wakeup = waitqueue_active(&pkmap_map_wait);
+ }
+ spin_unlock(&kmap_lock);
+
+ /* do wake-up, if needed, race-free outside of the spin lock */
+ if (need_wakeup)
+ wake_up(&pkmap_map_wait);
+}
+
+EXPORT_SYMBOL(kunmap_high);
+
+#define POOL_SIZE 64
+
+static __init int init_emergency_pool(void)
+{
+ struct sysinfo i;
+ si_meminfo(&i);
+ si_swapinfo(&i);
+
+ if (!i.totalhigh)
+ return 0;
+
+ page_pool = mempool_create(POOL_SIZE, page_pool_alloc, page_pool_free, NULL);
+ if (!page_pool)
+ BUG();
+ printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
+
+ return 0;
+}
+
+__initcall(init_emergency_pool);
+
+/*
+ * highmem version, map in to vec
+ */
+static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
+{
+ unsigned long flags;
+ unsigned char *vto;
+
+ local_irq_save(flags);
+ vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
+ memcpy(vto + to->bv_offset, vfrom, to->bv_len);
+ kunmap_atomic(vto, KM_BOUNCE_READ);
+ local_irq_restore(flags);
+}
+
+#else /* CONFIG_HIGHMEM */
+
+#define bounce_copy_vec(to, vfrom) \
+ memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
+
+#endif
+
+#define ISA_POOL_SIZE 16
+
+/*
+ * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
+ * as the max address, so check if the pool has already been created.
+ */
+int init_emergency_isa_pool(void)
+{
+ if (isa_page_pool)
+ return 0;
+
+ isa_page_pool = mempool_create(ISA_POOL_SIZE, page_pool_alloc, page_pool_free, (void *) __GFP_DMA);
+ if (!isa_page_pool)
+ BUG();
+
+ printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
+ return 0;
+}
+
+/*
+ * Simple bounce buffer support for highmem pages. Depending on the
+ * queue gfp mask set, *to may or may not be a highmem page. kmap it
+ * always, it will do the Right Thing
+ */
+static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
+{
+ unsigned char *vfrom;
+ struct bio_vec *tovec, *fromvec;
+ int i;
+
+ __bio_for_each_segment(tovec, to, i, 0) {
+ fromvec = from->bi_io_vec + i;
+
+ /*
+ * not bounced
+ */
+ if (tovec->bv_page == fromvec->bv_page)
+ continue;
+
+ /*
+ * fromvec->bv_offset and fromvec->bv_len might have been
+ * modified by the block layer, so use the original copy,
+ * bounce_copy_vec already uses tovec->bv_len
+ */
+ vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
+
+ flush_dcache_page(tovec->bv_page);
+ bounce_copy_vec(tovec, vfrom);
+ }
+}
+
+static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
+{
+ struct bio *bio_orig = bio->bi_private;
+ struct bio_vec *bvec, *org_vec;
+ int i;
+
+ if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
+ set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
+
+ /*
+ * free up bounce indirect pages used
+ */
+ __bio_for_each_segment(bvec, bio, i, 0) {
+ org_vec = bio_orig->bi_io_vec + i;
+ if (bvec->bv_page == org_vec->bv_page)
+ continue;
+
+ mempool_free(bvec->bv_page, pool);
+ }
+
+ bio_endio(bio_orig, bio_orig->bi_size, err);
+ bio_put(bio);
+}
+
+static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done,int err)
+{
+ if (bio->bi_size)
+ return 1;
+
+ bounce_end_io(bio, page_pool, err);
+ return 0;
+}
+
+static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err)
+{
+ if (bio->bi_size)
+ return 1;
+
+ bounce_end_io(bio, isa_page_pool, err);
+ return 0;
+}
+
+static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err)
+{
+ struct bio *bio_orig = bio->bi_private;
+
+ if (test_bit(BIO_UPTODATE, &bio->bi_flags))
+ copy_to_high_bio_irq(bio_orig, bio);
+
+ bounce_end_io(bio, pool, err);
+}
+
+static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
+{
+ if (bio->bi_size)
+ return 1;
+
+ __bounce_end_io_read(bio, page_pool, err);
+ return 0;
+}
+
+static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err)
+{
+ if (bio->bi_size)
+ return 1;
+
+ __bounce_end_io_read(bio, isa_page_pool, err);
+ return 0;
+}
+
+static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig,
+ mempool_t *pool)
+{
+ struct page *page;
+ struct bio *bio = NULL;
+ int i, rw = bio_data_dir(*bio_orig);
+ struct bio_vec *to, *from;
+
+ bio_for_each_segment(from, *bio_orig, i) {
+ page = from->bv_page;
+
+ /*
+ * is destination page below bounce pfn?
+ */
+ if (page_to_pfn(page) < q->bounce_pfn)
+ continue;
+
+ /*
+ * irk, bounce it
+ */
+ if (!bio)
+ bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
+
+ to = bio->bi_io_vec + i;
+
+ to->bv_page = mempool_alloc(pool, q->bounce_gfp);
+ to->bv_len = from->bv_len;
+ to->bv_offset = from->bv_offset;
+
+ if (rw == WRITE) {
+ char *vto, *vfrom;
+
+ flush_dcache_page(from->bv_page);
+ vto = page_address(to->bv_page) + to->bv_offset;
+ vfrom = kmap(from->bv_page) + from->bv_offset;
+ memcpy(vto, vfrom, to->bv_len);
+ kunmap(from->bv_page);
+ }
+ }
+
+ /*
+ * no pages bounced
+ */
+ if (!bio)
+ return;
+
+ /*
+ * at least one page was bounced, fill in possible non-highmem
+ * pages
+ */
+ bio_for_each_segment(from, *bio_orig, i) {
+ to = bio_iovec_idx(bio, i);
+ if (!to->bv_page) {
+ to->bv_page = from->bv_page;
+ to->bv_len = from->bv_len;
+ to->bv_offset = from->bv_offset;
+ }
+ }
+
+ bio->bi_bdev = (*bio_orig)->bi_bdev;
+ bio->bi_flags |= (1 << BIO_BOUNCED);
+ bio->bi_sector = (*bio_orig)->bi_sector;
+ bio->bi_rw = (*bio_orig)->bi_rw;
+
+ bio->bi_vcnt = (*bio_orig)->bi_vcnt;
+ bio->bi_idx = (*bio_orig)->bi_idx;
+ bio->bi_size = (*bio_orig)->bi_size;
+
+ if (pool == page_pool) {
+ bio->bi_end_io = bounce_end_io_write;
+ if (rw == READ)
+ bio->bi_end_io = bounce_end_io_read;
+ } else {
+ bio->bi_end_io = bounce_end_io_write_isa;
+ if (rw == READ)
+ bio->bi_end_io = bounce_end_io_read_isa;
+ }
+
+ bio->bi_private = *bio_orig;
+ *bio_orig = bio;
+}
+
+void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig)
+{
+ mempool_t *pool;
+
+ /*
+ * for non-isa bounce case, just check if the bounce pfn is equal
+ * to or bigger than the highest pfn in the system -- in that case,
+ * don't waste time iterating over bio segments
+ */
+ if (!(q->bounce_gfp & GFP_DMA)) {
+ if (q->bounce_pfn >= blk_max_pfn)
+ return;
+ pool = page_pool;
+ } else {
+ BUG_ON(!isa_page_pool);
+ pool = isa_page_pool;
+ }
+
+ /*
+ * slow path
+ */
+ __blk_queue_bounce(q, bio_orig, pool);
+}
+
+EXPORT_SYMBOL(blk_queue_bounce);
+
+#if defined(HASHED_PAGE_VIRTUAL)
+
+#define PA_HASH_ORDER 7
+
+/*
+ * Describes one page->virtual association
+ */
+struct page_address_map {
+ struct page *page;
+ void *virtual;
+ struct list_head list;
+};
+
+/*
+ * page_address_map freelist, allocated from page_address_maps.
+ */
+static struct list_head page_address_pool; /* freelist */
+static spinlock_t pool_lock; /* protects page_address_pool */
+
+/*
+ * Hash table bucket
+ */
+static struct page_address_slot {
+ struct list_head lh; /* List of page_address_maps */
+ spinlock_t lock; /* Protect this bucket's list */
+} ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER];
+
+static struct page_address_slot *page_slot(struct page *page)
+{
+ return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)];
+}
+
+void *page_address(struct page *page)
+{
+ unsigned long flags;
+ void *ret;
+ struct page_address_slot *pas;
+
+ if (!PageHighMem(page))
+ return lowmem_page_address(page);
+
+ pas = page_slot(page);
+ ret = NULL;
+ spin_lock_irqsave(&pas->lock, flags);
+ if (!list_empty(&pas->lh)) {
+ struct page_address_map *pam;
+
+ list_for_each_entry(pam, &pas->lh, list) {
+ if (pam->page == page) {
+ ret = pam->virtual;
+ goto done;
+ }
+ }
+ }
+done:
+ spin_unlock_irqrestore(&pas->lock, flags);
+ return ret;
+}
+
+EXPORT_SYMBOL(page_address);
+
+void set_page_address(struct page *page, void *virtual)
+{
+ unsigned long flags;
+ struct page_address_slot *pas;
+ struct page_address_map *pam;
+
+ BUG_ON(!PageHighMem(page));
+
+ pas = page_slot(page);
+ if (virtual) { /* Add */
+ BUG_ON(list_empty(&page_address_pool));
+
+ spin_lock_irqsave(&pool_lock, flags);
+ pam = list_entry(page_address_pool.next,
+ struct page_address_map, list);
+ list_del(&pam->list);
+ spin_unlock_irqrestore(&pool_lock, flags);
+
+ pam->page = page;
+ pam->virtual = virtual;
+
+ spin_lock_irqsave(&pas->lock, flags);
+ list_add_tail(&pam->list, &pas->lh);
+ spin_unlock_irqrestore(&pas->lock, flags);
+ } else { /* Remove */
+ spin_lock_irqsave(&pas->lock, flags);
+ list_for_each_entry(pam, &pas->lh, list) {
+ if (pam->page == page) {
+ list_del(&pam->list);
+ spin_unlock_irqrestore(&pas->lock, flags);
+ spin_lock_irqsave(&pool_lock, flags);
+ list_add_tail(&pam->list, &page_address_pool);
+ spin_unlock_irqrestore(&pool_lock, flags);
+ goto done;
+ }
+ }
+ spin_unlock_irqrestore(&pas->lock, flags);
+ }
+done:
+ return;
+}
+
+static struct page_address_map page_address_maps[LAST_PKMAP];
+
+void __init page_address_init(void)
+{
+ int i;
+
+ INIT_LIST_HEAD(&page_address_pool);
+ for (i = 0; i < ARRAY_SIZE(page_address_maps); i++)
+ list_add(&page_address_maps[i].list, &page_address_pool);
+ for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) {
+ INIT_LIST_HEAD(&page_address_htable[i].lh);
+ spin_lock_init(&page_address_htable[i].lock);
+ }
+ spin_lock_init(&pool_lock);
+}
+
+#endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */
free_one_pgd(tlb, page_dir);
page_dir++;
} while (--nr);
+#ifdef CONFIG_XEN_BATCH_MODE2
+ XEN_flush_page_update_queue();
+#endif
+
}
pte_t fastcall * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
* in the parent and the child
*/
if (cow) {
+#ifdef CONFIG_XEN_BATCH_MODE2
+/* XEN modification: modified ordering here to avoid RaW hazard. */
+ pte = *src_pte;
+ pte = pte_wrprotect(pte);
+ ptep_set_wrprotect(src_pte);
+#else
ptep_set_wrprotect(src_pte);
pte = *src_pte;
+#endif
}
/*
unlock_page(page);
flush_icache_page(vma, page);
+
+#ifdef CONFIG_XEN_BATCH_MODE2
+ if ( likely(vma->vm_mm == current->mm) ) {
+ XEN_flush_page_update_queue();
+ HYPERVISOR_update_va_mapping(address, pte, 0);
+ } else {
+ set_pte(page_table, pte);
+ XEN_flush_page_update_queue();
+ }
+#else
set_pte(page_table, pte);
+#endif
+
+
page_add_anon_rmap(page, vma, address);
if (write_access) {
page_add_anon_rmap(page, vma, addr);
}
+#ifdef CONFIG_XEN_BATCH_MODE2
+ if ( likely(vma->vm_mm == current->mm) ) {
+ XEN_flush_page_update_queue();
+ HYPERVISOR_update_va_mapping(addr, entry, 0);
+ } else {
+ set_pte(page_table, entry);
+ XEN_flush_page_update_queue();
+ }
+#else
ptep_establish_new(vma, addr, page_table, entry);
+#endif
pte_unmap(page_table);
/* No need to invalidate - it was non-present before */
entry = mk_pte(new_page, vma->vm_page_prot);
if (write_access)
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
+#ifdef CONFIG_XEN_BATCH_MODE2
+ if ( likely(vma->vm_mm == current->mm) ) {
+ XEN_flush_page_update_queue();
+ HYPERVISOR_update_va_mapping(address, entry, 0);
+ } else {
+ set_pte(page_table, entry);
+ XEN_flush_page_update_queue();
+ }
+#else
ptep_establish_new(vma, address, page_table, entry);
+#endif
if (anon) {
lru_cache_add_active(new_page);
page_add_anon_rmap(new_page, vma, address);
--- /dev/null
+/*
+ * linux/mm/swapfile.c
+ *
+ * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
+ * Swap reorganised 29.12.95, Stephen Tweedie
+ */
+
+#include <linux/config.h>
+#include <linux/mm.h>
+#include <linux/hugetlb.h>
+#include <linux/mman.h>
+#include <linux/slab.h>
+#include <linux/kernel_stat.h>
+#include <linux/swap.h>
+#include <linux/vmalloc.h>
+#include <linux/pagemap.h>
+#include <linux/namei.h>
+#include <linux/shm.h>
+#include <linux/blkdev.h>
+#include <linux/writeback.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/rmap.h>
+#include <linux/security.h>
+#include <linux/backing-dev.h>
+#include <linux/syscalls.h>
+
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+#include <linux/swapops.h>
+
+spinlock_t swaplock = SPIN_LOCK_UNLOCKED;
+unsigned int nr_swapfiles;
+long total_swap_pages;
+static int swap_overflow;
+
+EXPORT_SYMBOL(total_swap_pages);
+
+static const char Bad_file[] = "Bad swap file entry ";
+static const char Unused_file[] = "Unused swap file entry ";
+static const char Bad_offset[] = "Bad swap offset entry ";
+static const char Unused_offset[] = "Unused swap offset entry ";
+
+struct swap_list_t swap_list = {-1, -1};
+
+struct swap_info_struct swap_info[MAX_SWAPFILES];
+
+static DECLARE_MUTEX(swapon_sem);
+
+/*
+ * We need this because the bdev->unplug_fn can sleep and we cannot
+ * hold swap_list_lock while calling the unplug_fn. And swap_list_lock
+ * cannot be turned into a semaphore.
+ */
+static DECLARE_RWSEM(swap_unplug_sem);
+
+#define SWAPFILE_CLUSTER 256
+
+void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page)
+{
+ swp_entry_t entry;
+
+ down_read(&swap_unplug_sem);
+ entry.val = page->private;
+ if (PageSwapCache(page)) {
+ struct block_device *bdev = swap_info[swp_type(entry)].bdev;
+ struct backing_dev_info *bdi;
+
+ /*
+ * If the page is removed from swapcache from under us (with a
+ * racy try_to_unuse/swapoff) we need an additional reference
+ * count to avoid reading garbage from page->private above. If
+ * the WARN_ON triggers during a swapoff it maybe the race
+ * condition and it's harmless. However if it triggers without
+ * swapoff it signals a problem.
+ */
+ WARN_ON(page_count(page) <= 1);
+
+ bdi = bdev->bd_inode->i_mapping->backing_dev_info;
+ bdi->unplug_io_fn(bdi, page);
+ }
+ up_read(&swap_unplug_sem);
+}
+
+static inline int scan_swap_map(struct swap_info_struct *si)
+{
+ unsigned long offset;
+ /*
+ * We try to cluster swap pages by allocating them
+ * sequentially in swap. Once we've allocated
+ * SWAPFILE_CLUSTER pages this way, however, we resort to
+ * first-free allocation, starting a new cluster. This
+ * prevents us from scattering swap pages all over the entire
+ * swap partition, so that we reduce overall disk seek times
+ * between swap pages. -- sct */
+ if (si->cluster_nr) {
+ while (si->cluster_next <= si->highest_bit) {
+ offset = si->cluster_next++;
+ if (si->swap_map[offset])
+ continue;
+ si->cluster_nr--;
+ goto got_page;
+ }
+ }
+ si->cluster_nr = SWAPFILE_CLUSTER;
+
+ /* try to find an empty (even not aligned) cluster. */
+ offset = si->lowest_bit;
+ check_next_cluster:
+ if (offset+SWAPFILE_CLUSTER-1 <= si->highest_bit)
+ {
+ unsigned long nr;
+ for (nr = offset; nr < offset+SWAPFILE_CLUSTER; nr++)
+ if (si->swap_map[nr])
+ {
+ offset = nr+1;
+ goto check_next_cluster;
+ }
+ /* We found a completly empty cluster, so start
+ * using it.
+ */
+ goto got_page;
+ }
+ /* No luck, so now go finegrined as usual. -Andrea */
+ for (offset = si->lowest_bit; offset <= si->highest_bit ; offset++) {
+ if (si->swap_map[offset])
+ continue;
+ si->lowest_bit = offset+1;
+ got_page:
+ if (offset == si->lowest_bit)
+ si->lowest_bit++;
+ if (offset == si->highest_bit)
+ si->highest_bit--;
+ if (si->lowest_bit > si->highest_bit) {
+ si->lowest_bit = si->max;
+ si->highest_bit = 0;
+ }
+ si->swap_map[offset] = 1;
+ si->inuse_pages++;
+ nr_swap_pages--;
+ si->cluster_next = offset+1;
+ return offset;
+ }
+ si->lowest_bit = si->max;
+ si->highest_bit = 0;
+ return 0;
+}
+
+swp_entry_t get_swap_page(void)
+{
+ struct swap_info_struct * p;
+ unsigned long offset;
+ swp_entry_t entry;
+ int type, wrapped = 0;
+
+ entry.val = 0; /* Out of memory */
+ swap_list_lock();
+ type = swap_list.next;
+ if (type < 0)
+ goto out;
+ if (nr_swap_pages <= 0)
+ goto out;
+
+ while (1) {
+ p = &swap_info[type];
+ if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
+ swap_device_lock(p);
+ offset = scan_swap_map(p);
+ swap_device_unlock(p);
+ if (offset) {
+ entry = swp_entry(type,offset);
+ type = swap_info[type].next;
+ if (type < 0 ||
+ p->prio != swap_info[type].prio) {
+ swap_list.next = swap_list.head;
+ } else {
+ swap_list.next = type;
+ }
+ goto out;
+ }
+ }
+ type = p->next;
+ if (!wrapped) {
+ if (type < 0 || p->prio != swap_info[type].prio) {
+ type = swap_list.head;
+ wrapped = 1;
+ }
+ } else
+ if (type < 0)
+ goto out; /* out of swap space */
+ }
+out:
+ swap_list_unlock();
+ return entry;
+}
+
+static struct swap_info_struct * swap_info_get(swp_entry_t entry)
+{
+ struct swap_info_struct * p;
+ unsigned long offset, type;
+
+ if (!entry.val)
+ goto out;
+ type = swp_type(entry);
+ if (type >= nr_swapfiles)
+ goto bad_nofile;
+ p = & swap_info[type];
+ if (!(p->flags & SWP_USED))
+ goto bad_device;
+ offset = swp_offset(entry);
+ if (offset >= p->max)
+ goto bad_offset;
+ if (!p->swap_map[offset])
+ goto bad_free;
+ swap_list_lock();
+ if (p->prio > swap_info[swap_list.next].prio)
+ swap_list.next = type;
+ swap_device_lock(p);
+ return p;
+
+bad_free:
+ printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val);
+ goto out;
+bad_offset:
+ printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val);
+ goto out;
+bad_device:
+ printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val);
+ goto out;
+bad_nofile:
+ printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val);
+out:
+ return NULL;
+}
+
+static void swap_info_put(struct swap_info_struct * p)
+{
+ swap_device_unlock(p);
+ swap_list_unlock();
+}
+
+static int swap_entry_free(struct swap_info_struct *p, unsigned long offset)
+{
+ int count = p->swap_map[offset];
+
+ if (count < SWAP_MAP_MAX) {
+ count--;
+ p->swap_map[offset] = count;
+ if (!count) {
+ if (offset < p->lowest_bit)
+ p->lowest_bit = offset;
+ if (offset > p->highest_bit)
+ p->highest_bit = offset;
+ nr_swap_pages++;
+ p->inuse_pages--;
+ }
+ }
+ return count;
+}
+
+/*
+ * Caller has made sure that the swapdevice corresponding to entry
+ * is still around or has not been recycled.
+ */
+void swap_free(swp_entry_t entry)
+{
+ struct swap_info_struct * p;
+
+ p = swap_info_get(entry);
+ if (p) {
+ swap_entry_free(p, swp_offset(entry));
+ swap_info_put(p);
+ }
+}
+
+/*
+ * Check if we're the only user of a swap page,
+ * when the page is locked.
+ */
+static int exclusive_swap_page(struct page *page)
+{
+ int retval = 0;
+ struct swap_info_struct * p;
+ swp_entry_t entry;
+
+ entry.val = page->private;
+ p = swap_info_get(entry);
+ if (p) {
+ /* Is the only swap cache user the cache itself? */
+ if (p->swap_map[swp_offset(entry)] == 1) {
+ /* Recheck the page count with the swapcache lock held.. */
+ spin_lock_irq(&swapper_space.tree_lock);
+ if (page_count(page) == 2)
+ retval = 1;
+ spin_unlock_irq(&swapper_space.tree_lock);
+ }
+ swap_info_put(p);
+ }
+ return retval;
+}
+
+/*
+ * We can use this swap cache entry directly
+ * if there are no other references to it.
+ *
+ * Here "exclusive_swap_page()" does the real
+ * work, but we opportunistically check whether
+ * we need to get all the locks first..
+ */
+int can_share_swap_page(struct page *page)
+{
+ int retval = 0;
+
+ if (!PageLocked(page))
+ BUG();
+ switch (page_count(page)) {
+ case 3:
+ if (!PagePrivate(page))
+ break;
+ /* Fallthrough */
+ case 2:
+ if (!PageSwapCache(page))
+ break;
+ retval = exclusive_swap_page(page);
+ break;
+ case 1:
+ if (PageReserved(page))
+ break;
+ retval = 1;
+ }
+ return retval;
+}
+
+/*
+ * Work out if there are any other processes sharing this
+ * swap cache page. Free it if you can. Return success.
+ */
+int remove_exclusive_swap_page(struct page *page)
+{
+ int retval;
+ struct swap_info_struct * p;
+ swp_entry_t entry;
+
+ BUG_ON(PagePrivate(page));
+ BUG_ON(!PageLocked(page));
+
+ if (!PageSwapCache(page))
+ return 0;
+ if (PageWriteback(page))
+ return 0;
+ if (page_count(page) != 2) /* 2: us + cache */
+ return 0;
+
+ entry.val = page->private;
+ p = swap_info_get(entry);
+ if (!p)
+ return 0;
+
+ /* Is the only swap cache user the cache itself? */
+ retval = 0;
+ if (p->swap_map[swp_offset(entry)] == 1) {
+ /* Recheck the page count with the swapcache lock held.. */
+ spin_lock_irq(&swapper_space.tree_lock);
+ if ((page_count(page) == 2) && !PageWriteback(page)) {
+ __delete_from_swap_cache(page);
+ SetPageDirty(page);
+ retval = 1;
+ }
+ spin_unlock_irq(&swapper_space.tree_lock);
+ }
+ swap_info_put(p);
+
+ if (retval) {
+ swap_free(entry);
+ page_cache_release(page);
+ }
+
+ return retval;
+}
+
+/*
+ * Free the swap entry like above, but also try to
+ * free the page cache entry if it is the last user.
+ */
+void free_swap_and_cache(swp_entry_t entry)
+{
+ struct swap_info_struct * p;
+ struct page *page = NULL;
+
+ p = swap_info_get(entry);
+ if (p) {
+ if (swap_entry_free(p, swp_offset(entry)) == 1) {
+ spin_lock_irq(&swapper_space.tree_lock);
+ page = radix_tree_lookup(&swapper_space.page_tree,
+ entry.val);
+ if (page && TestSetPageLocked(page))
+ page = NULL;
+ spin_unlock_irq(&swapper_space.tree_lock);
+ }
+ swap_info_put(p);
+ }
+ if (page) {
+ int one_user;
+
+ BUG_ON(PagePrivate(page));
+ page_cache_get(page);
+ one_user = (page_count(page) == 2);
+ /* Only cache user (+us), or swap space full? Free it! */
+ if (!PageWriteback(page) && (one_user || vm_swap_full())) {
+ delete_from_swap_cache(page);
+ SetPageDirty(page);
+ }
+ unlock_page(page);
+ page_cache_release(page);
+ }
+}
+
+/*
+ * The swap entry has been read in advance, and we return 1 to indicate
+ * that the page has been used or is no longer needed.
+ *
+ * Always set the resulting pte to be nowrite (the same as COW pages
+ * after one process has exited). We don't know just how many PTEs will
+ * share this swap entry, so be cautious and let do_wp_page work out
+ * what to do if a write is requested later.
+ */
+/* vma->vm_mm->page_table_lock is held */
+static void
+unuse_pte(struct vm_area_struct *vma, unsigned long address, pte_t *dir,
+ swp_entry_t entry, struct page *page)
+{
+ vma->vm_mm->rss++;
+ get_page(page);
+ set_pte(dir, pte_mkold(mk_pte(page, vma->vm_page_prot)));
+ page_add_anon_rmap(page, vma, address);
+ swap_free(entry);
+}
+
+/* vma->vm_mm->page_table_lock is held */
+static unsigned long unuse_pmd(struct vm_area_struct * vma, pmd_t *dir,
+ unsigned long address, unsigned long size, unsigned long offset,
+ swp_entry_t entry, struct page *page)
+{
+ pte_t * pte;
+ unsigned long end;
+ pte_t swp_pte = swp_entry_to_pte(entry);
+
+ if (pmd_none(*dir))
+ return 0;
+ if (pmd_bad(*dir)) {
+ pmd_ERROR(*dir);
+ pmd_clear(dir);
+ return 0;
+ }
+ pte = pte_offset_map(dir, address);
+ offset += address & PMD_MASK;
+ address &= ~PMD_MASK;
+ end = address + size;
+ if (end > PMD_SIZE)
+ end = PMD_SIZE;
+ do {
+ /*
+ * swapoff spends a _lot_ of time in this loop!
+ * Test inline before going to call unuse_pte.
+ */
+ if (unlikely(pte_same(*pte, swp_pte))) {
+ unuse_pte(vma, offset + address, pte, entry, page);
+ pte_unmap(pte);
+
+ /*
+ * Move the page to the active list so it is not
+ * immediately swapped out again after swapon.
+ */
+ activate_page(page);
+
+ /* add 1 since address may be 0 */
+ return 1 + offset + address;
+ }
+ address += PAGE_SIZE;
+ pte++;
+ } while (address && (address < end));
+ pte_unmap(pte - 1);
+ return 0;
+}
+
+/* vma->vm_mm->page_table_lock is held */
+static unsigned long unuse_pgd(struct vm_area_struct * vma, pgd_t *dir,
+ unsigned long address, unsigned long size,
+ swp_entry_t entry, struct page *page)
+{
+ pmd_t * pmd;
+ unsigned long offset, end;
+ unsigned long foundaddr;
+
+ if (pgd_none(*dir))
+ return 0;
+ if (pgd_bad(*dir)) {
+ pgd_ERROR(*dir);
+ pgd_clear(dir);
+ return 0;
+ }
+ pmd = pmd_offset(dir, address);
+ offset = address & PGDIR_MASK;
+ address &= ~PGDIR_MASK;
+ end = address + size;
+ if (end > PGDIR_SIZE)
+ end = PGDIR_SIZE;
+ if (address >= end)
+ BUG();
+ do {
+ foundaddr = unuse_pmd(vma, pmd, address, end - address,
+ offset, entry, page);
+ if (foundaddr)
+ return foundaddr;
+ address = (address + PMD_SIZE) & PMD_MASK;
+ pmd++;
+ } while (address && (address < end));
+ return 0;
+}
+
+/* vma->vm_mm->page_table_lock is held */
+static unsigned long unuse_vma(struct vm_area_struct * vma,
+ swp_entry_t entry, struct page *page)
+{
+ pgd_t *pgdir;
+ unsigned long start, end;
+ unsigned long foundaddr;
+
+ if (page->mapping) {
+ start = page_address_in_vma(page, vma);
+ if (start == -EFAULT)
+ return 0;
+ else
+ end = start + PAGE_SIZE;
+ } else {
+ start = vma->vm_start;
+ end = vma->vm_end;
+ }
+ pgdir = pgd_offset(vma->vm_mm, start);
+ do {
+ foundaddr = unuse_pgd(vma, pgdir, start, end - start,
+ entry, page);
+ if (foundaddr)
+ return foundaddr;
+ start = (start + PGDIR_SIZE) & PGDIR_MASK;
+ pgdir++;
+ } while (start && (start < end));
+ return 0;
+}
+
+static int unuse_process(struct mm_struct * mm,
+ swp_entry_t entry, struct page* page)
+{
+ struct vm_area_struct* vma;
+ unsigned long foundaddr = 0;
+
+ /*
+ * Go through process' page directory.
+ */
+ if (!down_read_trylock(&mm->mmap_sem)) {
+ /*
+ * Our reference to the page stops try_to_unmap_one from
+ * unmapping its ptes, so swapoff can make progress.
+ */
+ unlock_page(page);
+ down_read(&mm->mmap_sem);
+ lock_page(page);
+ }
+ spin_lock(&mm->page_table_lock);
+ for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ if (vma->anon_vma) {
+ foundaddr = unuse_vma(vma, entry, page);
+ if (foundaddr)
+ break;
+ }
+ }
+#ifdef CONFIG_XEN_BATCH_MODE2
+ XEN_flush_page_update_queue();
+#endif
+ spin_unlock(&mm->page_table_lock);
+ up_read(&mm->mmap_sem);
+ /*
+ * Currently unuse_process cannot fail, but leave error handling
+ * at call sites for now, since we change it from time to time.
+ */
+ return 0;
+}
+
+/*
+ * Scan swap_map from current position to next entry still in use.
+ * Recycle to start on reaching the end, returning 0 when empty.
+ */
+static int find_next_to_unuse(struct swap_info_struct *si, int prev)
+{
+ int max = si->max;
+ int i = prev;
+ int count;
+
+ /*
+ * No need for swap_device_lock(si) here: we're just looking
+ * for whether an entry is in use, not modifying it; false
+ * hits are okay, and sys_swapoff() has already prevented new
+ * allocations from this area (while holding swap_list_lock()).
+ */
+ for (;;) {
+ if (++i >= max) {
+ if (!prev) {
+ i = 0;
+ break;
+ }
+ /*
+ * No entries in use at top of swap_map,
+ * loop back to start and recheck there.
+ */
+ max = prev + 1;
+ prev = 0;
+ i = 1;
+ }
+ count = si->swap_map[i];
+ if (count && count != SWAP_MAP_BAD)
+ break;
+ }
+ return i;
+}
+
+/*
+ * We completely avoid races by reading each swap page in advance,
+ * and then search for the process using it. All the necessary
+ * page table adjustments can then be made atomically.
+ */
+static int try_to_unuse(unsigned int type)
+{
+ struct swap_info_struct * si = &swap_info[type];
+ struct mm_struct *start_mm;
+ unsigned short *swap_map;
+ unsigned short swcount;
+ struct page *page;
+ swp_entry_t entry;
+ int i = 0;
+ int retval = 0;
+ int reset_overflow = 0;
+ int shmem;
+
+ /*
+ * When searching mms for an entry, a good strategy is to
+ * start at the first mm we freed the previous entry from
+ * (though actually we don't notice whether we or coincidence
+ * freed the entry). Initialize this start_mm with a hold.
+ *
+ * A simpler strategy would be to start at the last mm we
+ * freed the previous entry from; but that would take less
+ * advantage of mmlist ordering, which clusters forked mms
+ * together, child after parent. If we race with dup_mmap(), we
+ * prefer to resolve parent before child, lest we miss entries
+ * duplicated after we scanned child: using last mm would invert
+ * that. Though it's only a serious concern when an overflowed
+ * swap count is reset from SWAP_MAP_MAX, preventing a rescan.
+ */
+ start_mm = &init_mm;
+ atomic_inc(&init_mm.mm_users);
+
+ /*
+ * Keep on scanning until all entries have gone. Usually,
+ * one pass through swap_map is enough, but not necessarily:
+ * there are races when an instance of an entry might be missed.
+ */
+ while ((i = find_next_to_unuse(si, i)) != 0) {
+ if (signal_pending(current)) {
+ retval = -EINTR;
+ break;
+ }
+
+ /*
+ * Get a page for the entry, using the existing swap
+ * cache page if there is one. Otherwise, get a clean
+ * page and read the swap into it.
+ */
+ swap_map = &si->swap_map[i];
+ entry = swp_entry(type, i);
+ page = read_swap_cache_async(entry, NULL, 0);
+ if (!page) {
+ /*
+ * Either swap_duplicate() failed because entry
+ * has been freed independently, and will not be
+ * reused since sys_swapoff() already disabled
+ * allocation from here, or alloc_page() failed.
+ */
+ if (!*swap_map)
+ continue;
+ retval = -ENOMEM;
+ break;
+ }
+
+ /*
+ * Don't hold on to start_mm if it looks like exiting.
+ */
+ if (atomic_read(&start_mm->mm_users) == 1) {
+ mmput(start_mm);
+ start_mm = &init_mm;
+ atomic_inc(&init_mm.mm_users);
+ }
+
+ /*
+ * Wait for and lock page. When do_swap_page races with
+ * try_to_unuse, do_swap_page can handle the fault much
+ * faster than try_to_unuse can locate the entry. This
+ * apparently redundant "wait_on_page_locked" lets try_to_unuse
+ * defer to do_swap_page in such a case - in some tests,
+ * do_swap_page and try_to_unuse repeatedly compete.
+ */
+ wait_on_page_locked(page);
+ wait_on_page_writeback(page);
+ lock_page(page);
+ wait_on_page_writeback(page);
+
+ /*
+ * Remove all references to entry.
+ * Whenever we reach init_mm, there's no address space
+ * to search, but use it as a reminder to search shmem.
+ */
+ shmem = 0;
+ swcount = *swap_map;
+ if (swcount > 1) {
+ if (start_mm == &init_mm)
+ shmem = shmem_unuse(entry, page);
+ else
+ retval = unuse_process(start_mm, entry, page);
+ }
+ if (*swap_map > 1) {
+ int set_start_mm = (*swap_map >= swcount);
+ struct list_head *p = &start_mm->mmlist;
+ struct mm_struct *new_start_mm = start_mm;
+ struct mm_struct *prev_mm = start_mm;
+ struct mm_struct *mm;
+
+ atomic_inc(&new_start_mm->mm_users);
+ atomic_inc(&prev_mm->mm_users);
+ spin_lock(&mmlist_lock);
+ while (*swap_map > 1 && !retval &&
+ (p = p->next) != &start_mm->mmlist) {
+ mm = list_entry(p, struct mm_struct, mmlist);
+ if (atomic_inc_return(&mm->mm_users) == 1) {
+ atomic_dec(&mm->mm_users);
+ continue;
+ }
+ spin_unlock(&mmlist_lock);
+ mmput(prev_mm);
+ prev_mm = mm;
+
+ cond_resched();
+
+ swcount = *swap_map;
+ if (swcount <= 1)
+ ;
+ else if (mm == &init_mm) {
+ set_start_mm = 1;
+ shmem = shmem_unuse(entry, page);
+ } else
+ retval = unuse_process(mm, entry, page);
+ if (set_start_mm && *swap_map < swcount) {
+ mmput(new_start_mm);
+ atomic_inc(&mm->mm_users);
+ new_start_mm = mm;
+ set_start_mm = 0;
+ }
+ spin_lock(&mmlist_lock);
+ }
+ spin_unlock(&mmlist_lock);
+ mmput(prev_mm);
+ mmput(start_mm);
+ start_mm = new_start_mm;
+ }
+ if (retval) {
+ unlock_page(page);
+ page_cache_release(page);
+ break;
+ }
+
+ /*
+ * How could swap count reach 0x7fff when the maximum
+ * pid is 0x7fff, and there's no way to repeat a swap
+ * page within an mm (except in shmem, where it's the
+ * shared object which takes the reference count)?
+ * We believe SWAP_MAP_MAX cannot occur in Linux 2.4.
+ *
+ * If that's wrong, then we should worry more about
+ * exit_mmap() and do_munmap() cases described above:
+ * we might be resetting SWAP_MAP_MAX too early here.
+ * We know "Undead"s can happen, they're okay, so don't
+ * report them; but do report if we reset SWAP_MAP_MAX.
+ */
+ if (*swap_map == SWAP_MAP_MAX) {
+ swap_device_lock(si);
+ *swap_map = 1;
+ swap_device_unlock(si);
+ reset_overflow = 1;
+ }
+
+ /*
+ * If a reference remains (rare), we would like to leave
+ * the page in the swap cache; but try_to_unmap could
+ * then re-duplicate the entry once we drop page lock,
+ * so we might loop indefinitely; also, that page could
+ * not be swapped out to other storage meanwhile. So:
+ * delete from cache even if there's another reference,
+ * after ensuring that the data has been saved to disk -
+ * since if the reference remains (rarer), it will be
+ * read from disk into another page. Splitting into two
+ * pages would be incorrect if swap supported "shared
+ * private" pages, but they are handled by tmpfs files.
+ *
+ * Note shmem_unuse already deleted a swappage from
+ * the swap cache, unless the move to filepage failed:
+ * in which case it left swappage in cache, lowered its
+ * swap count to pass quickly through the loops above,
+ * and now we must reincrement count to try again later.
+ */
+ if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) {
+ struct writeback_control wbc = {
+ .sync_mode = WB_SYNC_NONE,
+ };
+
+ swap_writepage(page, &wbc);
+ lock_page(page);
+ wait_on_page_writeback(page);
+ }
+ if (PageSwapCache(page)) {
+ if (shmem)
+ swap_duplicate(entry);
+ else
+ delete_from_swap_cache(page);
+ }
+
+ /*
+ * So we could skip searching mms once swap count went
+ * to 1, we did not mark any present ptes as dirty: must
+ * mark page dirty so shrink_list will preserve it.
+ */
+ SetPageDirty(page);
+ unlock_page(page);
+ page_cache_release(page);
+
+ /*
+ * Make sure that we aren't completely killing
+ * interactive performance.
+ */
+ cond_resched();
+ }
+
+ mmput(start_mm);
+ if (reset_overflow) {
+ printk(KERN_WARNING "swapoff: cleared swap entry overflow\n");
+ swap_overflow = 0;
+ }
+ return retval;
+}
+
+/*
+ * After a successful try_to_unuse, if no swap is now in use, we know we
+ * can empty the mmlist. swap_list_lock must be held on entry and exit.
+ * Note that mmlist_lock nests inside swap_list_lock, and an mm must be
+ * added to the mmlist just after page_duplicate - before would be racy.
+ */
+static void drain_mmlist(void)
+{
+ struct list_head *p, *next;
+ unsigned int i;
+
+ for (i = 0; i < nr_swapfiles; i++)
+ if (swap_info[i].inuse_pages)
+ return;
+ spin_lock(&mmlist_lock);
+ list_for_each_safe(p, next, &init_mm.mmlist)
+ list_del_init(p);
+ spin_unlock(&mmlist_lock);
+}
+
+/*
+ * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
+ * corresponds to page offset `offset'.
+ */
+sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset)
+{
+ struct swap_extent *se = sis->curr_swap_extent;
+ struct swap_extent *start_se = se;
+
+ for ( ; ; ) {
+ struct list_head *lh;
+
+ if (se->start_page <= offset &&
+ offset < (se->start_page + se->nr_pages)) {
+ return se->start_block + (offset - se->start_page);
+ }
+ lh = se->list.prev;
+ if (lh == &sis->extent_list)
+ lh = lh->prev;
+ se = list_entry(lh, struct swap_extent, list);
+ sis->curr_swap_extent = se;
+ BUG_ON(se == start_se); /* It *must* be present */
+ }
+}
+
+/*
+ * Free all of a swapdev's extent information
+ */
+static void destroy_swap_extents(struct swap_info_struct *sis)
+{
+ while (!list_empty(&sis->extent_list)) {
+ struct swap_extent *se;
+
+ se = list_entry(sis->extent_list.next,
+ struct swap_extent, list);
+ list_del(&se->list);
+ kfree(se);
+ }
+ sis->nr_extents = 0;
+}
+
+/*
+ * Add a block range (and the corresponding page range) into this swapdev's
+ * extent list. The extent list is kept sorted in block order.
+ *
+ * This function rather assumes that it is called in ascending sector_t order.
+ * It doesn't look for extent coalescing opportunities.
+ */
+static int
+add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
+ unsigned long nr_pages, sector_t start_block)
+{
+ struct swap_extent *se;
+ struct swap_extent *new_se;
+ struct list_head *lh;
+
+ lh = sis->extent_list.next; /* The highest-addressed block */
+ while (lh != &sis->extent_list) {
+ se = list_entry(lh, struct swap_extent, list);
+ if (se->start_block + se->nr_pages == start_block &&
+ se->start_page + se->nr_pages == start_page) {
+ /* Merge it */
+ se->nr_pages += nr_pages;
+ return 0;
+ }
+ lh = lh->next;
+ }
+
+ /*
+ * No merge. Insert a new extent, preserving ordering.
+ */
+ new_se = kmalloc(sizeof(*se), GFP_KERNEL);
+ if (new_se == NULL)
+ return -ENOMEM;
+ new_se->start_page = start_page;
+ new_se->nr_pages = nr_pages;
+ new_se->start_block = start_block;
+
+ lh = sis->extent_list.prev; /* The lowest block */
+ while (lh != &sis->extent_list) {
+ se = list_entry(lh, struct swap_extent, list);
+ if (se->start_block > start_block)
+ break;
+ lh = lh->prev;
+ }
+ list_add_tail(&new_se->list, lh);
+ sis->nr_extents++;
+ return 0;
+}
+
+/*
+ * A `swap extent' is a simple thing which maps a contiguous range of pages
+ * onto a contiguous range of disk blocks. An ordered list of swap extents
+ * is built at swapon time and is then used at swap_writepage/swap_readpage
+ * time for locating where on disk a page belongs.
+ *
+ * If the swapfile is an S_ISBLK block device, a single extent is installed.
+ * This is done so that the main operating code can treat S_ISBLK and S_ISREG
+ * swap files identically.
+ *
+ * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
+ * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
+ * swapfiles are handled *identically* after swapon time.
+ *
+ * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
+ * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If
+ * some stray blocks are found which do not fall within the PAGE_SIZE alignment
+ * requirements, they are simply tossed out - we will never use those blocks
+ * for swapping.
+ *
+ * For S_ISREG swapfiles we hold i_sem across the life of the swapon. This
+ * prevents root from shooting her foot off by ftruncating an in-use swapfile,
+ * which will scribble on the fs.
+ *
+ * The amount of disk space which a single swap extent represents varies.
+ * Typically it is in the 1-4 megabyte range. So we can have hundreds of
+ * extents in the list. To avoid much list walking, we cache the previous
+ * search location in `curr_swap_extent', and start new searches from there.
+ * This is extremely effective. The average number of iterations in
+ * map_swap_page() has been measured at about 0.3 per page. - akpm.
+ */
+static int setup_swap_extents(struct swap_info_struct *sis)
+{
+ struct inode *inode;
+ unsigned blocks_per_page;
+ unsigned long page_no;
+ unsigned blkbits;
+ sector_t probe_block;
+ sector_t last_block;
+ int ret;
+
+ inode = sis->swap_file->f_mapping->host;
+ if (S_ISBLK(inode->i_mode)) {
+ ret = add_swap_extent(sis, 0, sis->max, 0);
+ goto done;
+ }
+
+ blkbits = inode->i_blkbits;
+ blocks_per_page = PAGE_SIZE >> blkbits;
+
+ /*
+ * Map all the blocks into the extent list. This code doesn't try
+ * to be very smart.
+ */
+ probe_block = 0;
+ page_no = 0;
+ last_block = i_size_read(inode) >> blkbits;
+ while ((probe_block + blocks_per_page) <= last_block &&
+ page_no < sis->max) {
+ unsigned block_in_page;
+ sector_t first_block;
+
+ first_block = bmap(inode, probe_block);
+ if (first_block == 0)
+ goto bad_bmap;
+
+ /*
+ * It must be PAGE_SIZE aligned on-disk
+ */
+ if (first_block & (blocks_per_page - 1)) {
+ probe_block++;
+ goto reprobe;
+ }
+
+ for (block_in_page = 1; block_in_page < blocks_per_page;
+ block_in_page++) {
+ sector_t block;
+
+ block = bmap(inode, probe_block + block_in_page);
+ if (block == 0)
+ goto bad_bmap;
+ if (block != first_block + block_in_page) {
+ /* Discontiguity */
+ probe_block++;
+ goto reprobe;
+ }
+ }
+
+ /*
+ * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
+ */
+ ret = add_swap_extent(sis, page_no, 1,
+ first_block >> (PAGE_SHIFT - blkbits));
+ if (ret)
+ goto out;
+ page_no++;
+ probe_block += blocks_per_page;
+reprobe:
+ continue;
+ }
+ ret = 0;
+ if (page_no == 0)
+ ret = -EINVAL;
+ sis->max = page_no;
+ sis->highest_bit = page_no - 1;
+done:
+ sis->curr_swap_extent = list_entry(sis->extent_list.prev,
+ struct swap_extent, list);
+ goto out;
+bad_bmap:
+ printk(KERN_ERR "swapon: swapfile has holes\n");
+ ret = -EINVAL;
+out:
+ return ret;
+}
+
+#if 0 /* We don't need this yet */
+#include <linux/backing-dev.h>
+int page_queue_congested(struct page *page)
+{
+ struct backing_dev_info *bdi;
+
+ BUG_ON(!PageLocked(page)); /* It pins the swap_info_struct */
+
+ if (PageSwapCache(page)) {
+ swp_entry_t entry = { .val = page->private };
+ struct swap_info_struct *sis;
+
+ sis = get_swap_info_struct(swp_type(entry));
+ bdi = sis->bdev->bd_inode->i_mapping->backing_dev_info;
+ } else
+ bdi = page->mapping->backing_dev_info;
+ return bdi_write_congested(bdi);
+}
+#endif
+
+asmlinkage long sys_swapoff(const char __user * specialfile)
+{
+ struct swap_info_struct * p = NULL;
+ unsigned short *swap_map;
+ struct file *swap_file, *victim;
+ struct address_space *mapping;
+ struct inode *inode;
+ char * pathname;
+ int i, type, prev;
+ int err;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ pathname = getname(specialfile);
+ err = PTR_ERR(pathname);
+ if (IS_ERR(pathname))
+ goto out;
+
+ victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0);
+ putname(pathname);
+ err = PTR_ERR(victim);
+ if (IS_ERR(victim))
+ goto out;
+
+ mapping = victim->f_mapping;
+ prev = -1;
+ swap_list_lock();
+ for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
+ p = swap_info + type;
+ if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
+ if (p->swap_file->f_mapping == mapping)
+ break;
+ }
+ prev = type;
+ }
+ if (type < 0) {
+ err = -EINVAL;
+ swap_list_unlock();
+ goto out_dput;
+ }
+ if (!security_vm_enough_memory(p->pages))
+ vm_unacct_memory(p->pages);
+ else {
+ err = -ENOMEM;
+ swap_list_unlock();
+ goto out_dput;
+ }
+ if (prev < 0) {
+ swap_list.head = p->next;
+ } else {
+ swap_info[prev].next = p->next;
+ }
+ if (type == swap_list.next) {
+ /* just pick something that's safe... */
+ swap_list.next = swap_list.head;
+ }
+ nr_swap_pages -= p->pages;
+ total_swap_pages -= p->pages;
+ p->flags &= ~SWP_WRITEOK;
+ swap_list_unlock();
+ current->flags |= PF_SWAPOFF;
+ err = try_to_unuse(type);
+ current->flags &= ~PF_SWAPOFF;
+
+ /* wait for any unplug function to finish */
+ down_write(&swap_unplug_sem);
+ up_write(&swap_unplug_sem);
+
+ if (err) {
+ /* re-insert swap space back into swap_list */
+ swap_list_lock();
+ for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next)
+ if (p->prio >= swap_info[i].prio)
+ break;
+ p->next = i;
+ if (prev < 0)
+ swap_list.head = swap_list.next = p - swap_info;
+ else
+ swap_info[prev].next = p - swap_info;
+ nr_swap_pages += p->pages;
+ total_swap_pages += p->pages;
+ p->flags |= SWP_WRITEOK;
+ swap_list_unlock();
+ goto out_dput;
+ }
+ down(&swapon_sem);
+ swap_list_lock();
+ drain_mmlist();
+ swap_device_lock(p);
+ swap_file = p->swap_file;
+ p->swap_file = NULL;
+ p->max = 0;
+ swap_map = p->swap_map;
+ p->swap_map = NULL;
+ p->flags = 0;
+ destroy_swap_extents(p);
+ swap_device_unlock(p);
+ swap_list_unlock();
+ up(&swapon_sem);
+ vfree(swap_map);
+ inode = mapping->host;
+ if (S_ISBLK(inode->i_mode)) {
+ struct block_device *bdev = I_BDEV(inode);
+ set_blocksize(bdev, p->old_block_size);
+ bd_release(bdev);
+ } else {
+ down(&inode->i_sem);
+ inode->i_flags &= ~S_SWAPFILE;
+ up(&inode->i_sem);
+ }
+ filp_close(swap_file, NULL);
+ err = 0;
+
+out_dput:
+ filp_close(victim, NULL);
+out:
+ return err;
+}
+
+#ifdef CONFIG_PROC_FS
+/* iterator */
+static void *swap_start(struct seq_file *swap, loff_t *pos)
+{
+ struct swap_info_struct *ptr = swap_info;
+ int i;
+ loff_t l = *pos;
+
+ down(&swapon_sem);
+
+ for (i = 0; i < nr_swapfiles; i++, ptr++) {
+ if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
+ continue;
+ if (!l--)
+ return ptr;
+ }
+
+ return NULL;
+}
+
+static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
+{
+ struct swap_info_struct *ptr = v;
+ struct swap_info_struct *endptr = swap_info + nr_swapfiles;
+
+ for (++ptr; ptr < endptr; ptr++) {
+ if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
+ continue;
+ ++*pos;
+ return ptr;
+ }
+
+ return NULL;
+}
+
+static void swap_stop(struct seq_file *swap, void *v)
+{
+ up(&swapon_sem);
+}
+
+static int swap_show(struct seq_file *swap, void *v)
+{
+ struct swap_info_struct *ptr = v;
+ struct file *file;
+ int len;
+
+ if (v == swap_info)
+ seq_puts(swap, "Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
+
+ file = ptr->swap_file;
+ len = seq_path(swap, file->f_vfsmnt, file->f_dentry, " \t\n\\");
+ seq_printf(swap, "%*s%s\t%d\t%ld\t%d\n",
+ len < 40 ? 40 - len : 1, " ",
+ S_ISBLK(file->f_dentry->d_inode->i_mode) ?
+ "partition" : "file\t",
+ ptr->pages << (PAGE_SHIFT - 10),
+ ptr->inuse_pages << (PAGE_SHIFT - 10),
+ ptr->prio);
+ return 0;
+}
+
+static struct seq_operations swaps_op = {
+ .start = swap_start,
+ .next = swap_next,
+ .stop = swap_stop,
+ .show = swap_show
+};
+
+static int swaps_open(struct inode *inode, struct file *file)
+{
+ return seq_open(file, &swaps_op);
+}
+
+static struct file_operations proc_swaps_operations = {
+ .open = swaps_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+static int __init procswaps_init(void)
+{
+ struct proc_dir_entry *entry;
+
+ entry = create_proc_entry("swaps", 0, NULL);
+ if (entry)
+ entry->proc_fops = &proc_swaps_operations;
+ return 0;
+}
+__initcall(procswaps_init);
+#endif /* CONFIG_PROC_FS */
+
+/*
+ * Written 01/25/92 by Simmule Turner, heavily changed by Linus.
+ *
+ * The swapon system call
+ */
+asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
+{
+ struct swap_info_struct * p;
+ char *name = NULL;
+ struct block_device *bdev = NULL;
+ struct file *swap_file = NULL;
+ struct address_space *mapping;
+ unsigned int type;
+ int i, prev;
+ int error;
+ static int least_priority;
+ union swap_header *swap_header = NULL;
+ int swap_header_version;
+ int nr_good_pages = 0;
+ unsigned long maxpages = 1;
+ int swapfilesize;
+ unsigned short *swap_map;
+ struct page *page = NULL;
+ struct inode *inode = NULL;
+ int did_down = 0;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EPERM;
+ swap_list_lock();
+ p = swap_info;
+ for (type = 0 ; type < nr_swapfiles ; type++,p++)
+ if (!(p->flags & SWP_USED))
+ break;
+ error = -EPERM;
+ /*
+ * Test if adding another swap device is possible. There are
+ * two limiting factors: 1) the number of bits for the swap
+ * type swp_entry_t definition and 2) the number of bits for
+ * the swap type in the swap ptes as defined by the different
+ * architectures. To honor both limitations a swap entry
+ * with swap offset 0 and swap type ~0UL is created, encoded
+ * to a swap pte, decoded to a swp_entry_t again and finally
+ * the swap type part is extracted. This will mask all bits
+ * from the initial ~0UL that can't be encoded in either the
+ * swp_entry_t or the architecture definition of a swap pte.
+ */
+ if (type > swp_type(pte_to_swp_entry(swp_entry_to_pte(swp_entry(~0UL,0))))) {
+ swap_list_unlock();
+ goto out;
+ }
+ if (type >= nr_swapfiles)
+ nr_swapfiles = type+1;
+ INIT_LIST_HEAD(&p->extent_list);
+ p->flags = SWP_USED;
+ p->nr_extents = 0;
+ p->swap_file = NULL;
+ p->old_block_size = 0;
+ p->swap_map = NULL;
+ p->lowest_bit = 0;
+ p->highest_bit = 0;
+ p->cluster_nr = 0;
+ p->inuse_pages = 0;
+ spin_lock_init(&p->sdev_lock);
+ p->next = -1;
+ if (swap_flags & SWAP_FLAG_PREFER) {
+ p->prio =
+ (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT;
+ } else {
+ p->prio = --least_priority;
+ }
+ swap_list_unlock();
+ name = getname(specialfile);
+ error = PTR_ERR(name);
+ if (IS_ERR(name)) {
+ name = NULL;
+ goto bad_swap_2;
+ }
+ swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0);
+ error = PTR_ERR(swap_file);
+ if (IS_ERR(swap_file)) {
+ swap_file = NULL;
+ goto bad_swap_2;
+ }
+
+ p->swap_file = swap_file;
+ mapping = swap_file->f_mapping;
+ inode = mapping->host;
+
+ error = -EBUSY;
+ for (i = 0; i < nr_swapfiles; i++) {
+ struct swap_info_struct *q = &swap_info[i];
+
+ if (i == type || !q->swap_file)
+ continue;
+ if (mapping == q->swap_file->f_mapping)
+ goto bad_swap;
+ }
+
+ error = -EINVAL;
+ if (S_ISBLK(inode->i_mode)) {
+ bdev = I_BDEV(inode);
+ error = bd_claim(bdev, sys_swapon);
+ if (error < 0) {
+ bdev = NULL;
+ goto bad_swap;
+ }
+ p->old_block_size = block_size(bdev);
+ error = set_blocksize(bdev, PAGE_SIZE);
+ if (error < 0)
+ goto bad_swap;
+ p->bdev = bdev;
+ } else if (S_ISREG(inode->i_mode)) {
+ p->bdev = inode->i_sb->s_bdev;
+ down(&inode->i_sem);
+ did_down = 1;
+ if (IS_SWAPFILE(inode)) {
+ error = -EBUSY;
+ goto bad_swap;
+ }
+ } else {
+ goto bad_swap;
+ }
+
+ swapfilesize = i_size_read(inode) >> PAGE_SHIFT;
+
+ /*
+ * Read the swap header.
+ */
+ if (!mapping->a_ops->readpage) {
+ error = -EINVAL;
+ goto bad_swap;
+ }
+ page = read_cache_page(mapping, 0,
+ (filler_t *)mapping->a_ops->readpage, swap_file);
+ if (IS_ERR(page)) {
+ error = PTR_ERR(page);
+ goto bad_swap;
+ }
+ wait_on_page_locked(page);
+ if (!PageUptodate(page))
+ goto bad_swap;
+ kmap(page);
+ swap_header = page_address(page);
+
+ if (!memcmp("SWAP-SPACE",swap_header->magic.magic,10))
+ swap_header_version = 1;
+ else if (!memcmp("SWAPSPACE2",swap_header->magic.magic,10))
+ swap_header_version = 2;
+ else {
+ printk("Unable to find swap-space signature\n");
+ error = -EINVAL;
+ goto bad_swap;
+ }
+
+ switch (swap_header_version) {
+ case 1:
+ printk(KERN_ERR "version 0 swap is no longer supported. "
+ "Use mkswap -v1 %s\n", name);
+ error = -EINVAL;
+ goto bad_swap;
+ case 2:
+ /* Check the swap header's sub-version and the size of
+ the swap file and bad block lists */
+ if (swap_header->info.version != 1) {
+ printk(KERN_WARNING
+ "Unable to handle swap header version %d\n",
+ swap_header->info.version);
+ error = -EINVAL;
+ goto bad_swap;
+ }
+
+ p->lowest_bit = 1;
+ /*
+ * Find out how many pages are allowed for a single swap
+ * device. There are two limiting factors: 1) the number of
+ * bits for the swap offset in the swp_entry_t type and
+ * 2) the number of bits in the a swap pte as defined by
+ * the different architectures. In order to find the
+ * largest possible bit mask a swap entry with swap type 0
+ * and swap offset ~0UL is created, encoded to a swap pte,
+ * decoded to a swp_entry_t again and finally the swap
+ * offset is extracted. This will mask all the bits from
+ * the initial ~0UL mask that can't be encoded in either
+ * the swp_entry_t or the architecture definition of a
+ * swap pte.
+ */
+ maxpages = swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0,~0UL)))) - 1;
+ if (maxpages > swap_header->info.last_page)
+ maxpages = swap_header->info.last_page;
+ p->highest_bit = maxpages - 1;
+
+ error = -EINVAL;
+ if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
+ goto bad_swap;
+
+ /* OK, set up the swap map and apply the bad block list */
+ if (!(p->swap_map = vmalloc(maxpages * sizeof(short)))) {
+ error = -ENOMEM;
+ goto bad_swap;
+ }
+
+ error = 0;
+ memset(p->swap_map, 0, maxpages * sizeof(short));
+ for (i=0; i<swap_header->info.nr_badpages; i++) {
+ int page = swap_header->info.badpages[i];
+ if (page <= 0 || page >= swap_header->info.last_page)
+ error = -EINVAL;
+ else
+ p->swap_map[page] = SWAP_MAP_BAD;
+ }
+ nr_good_pages = swap_header->info.last_page -
+ swap_header->info.nr_badpages -
+ 1 /* header page */;
+ if (error)
+ goto bad_swap;
+ }
+
+ if (swapfilesize && maxpages > swapfilesize) {
+ printk(KERN_WARNING
+ "Swap area shorter than signature indicates\n");
+ error = -EINVAL;
+ goto bad_swap;
+ }
+ if (!nr_good_pages) {
+ printk(KERN_WARNING "Empty swap-file\n");
+ error = -EINVAL;
+ goto bad_swap;
+ }
+ p->swap_map[0] = SWAP_MAP_BAD;
+ p->max = maxpages;
+ p->pages = nr_good_pages;
+
+ error = setup_swap_extents(p);
+ if (error)
+ goto bad_swap;
+
+ down(&swapon_sem);
+ swap_list_lock();
+ swap_device_lock(p);
+ p->flags = SWP_ACTIVE;
+ nr_swap_pages += nr_good_pages;
+ total_swap_pages += nr_good_pages;
+ printk(KERN_INFO "Adding %dk swap on %s. Priority:%d extents:%d\n",
+ nr_good_pages<<(PAGE_SHIFT-10), name,
+ p->prio, p->nr_extents);
+
+ /* insert swap space into swap_list: */
+ prev = -1;
+ for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
+ if (p->prio >= swap_info[i].prio) {
+ break;
+ }
+ prev = i;
+ }
+ p->next = i;
+ if (prev < 0) {
+ swap_list.head = swap_list.next = p - swap_info;
+ } else {
+ swap_info[prev].next = p - swap_info;
+ }
+ swap_device_unlock(p);
+ swap_list_unlock();
+ up(&swapon_sem);
+ error = 0;
+ goto out;
+bad_swap:
+ if (bdev) {
+ set_blocksize(bdev, p->old_block_size);
+ bd_release(bdev);
+ }
+bad_swap_2:
+ swap_list_lock();
+ swap_map = p->swap_map;
+ p->swap_file = NULL;
+ p->swap_map = NULL;
+ p->flags = 0;
+ if (!(swap_flags & SWAP_FLAG_PREFER))
+ ++least_priority;
+ swap_list_unlock();
+ destroy_swap_extents(p);
+ if (swap_map)
+ vfree(swap_map);
+ if (swap_file)
+ filp_close(swap_file, NULL);
+out:
+ if (page && !IS_ERR(page)) {
+ kunmap(page);
+ page_cache_release(page);
+ }
+ if (name)
+ putname(name);
+ if (did_down) {
+ if (!error)
+ inode->i_flags |= S_SWAPFILE;
+ up(&inode->i_sem);
+ }
+ return error;
+}
+
+void si_swapinfo(struct sysinfo *val)
+{
+ unsigned int i;
+ unsigned long nr_to_be_unused = 0;
+
+ swap_list_lock();
+ for (i = 0; i < nr_swapfiles; i++) {
+ if (!(swap_info[i].flags & SWP_USED) ||
+ (swap_info[i].flags & SWP_WRITEOK))
+ continue;
+ nr_to_be_unused += swap_info[i].inuse_pages;
+ }
+ val->freeswap = nr_swap_pages + nr_to_be_unused;
+ val->totalswap = total_swap_pages + nr_to_be_unused;
+ swap_list_unlock();
+}
+
+/*
+ * Verify that a swap entry is valid and increment its swap map count.
+ *
+ * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
+ * "permanent", but will be reclaimed by the next swapoff.
+ */
+int swap_duplicate(swp_entry_t entry)
+{
+ struct swap_info_struct * p;
+ unsigned long offset, type;
+ int result = 0;
+
+ type = swp_type(entry);
+ if (type >= nr_swapfiles)
+ goto bad_file;
+ p = type + swap_info;
+ offset = swp_offset(entry);
+
+ swap_device_lock(p);
+ if (offset < p->max && p->swap_map[offset]) {
+ if (p->swap_map[offset] < SWAP_MAP_MAX - 1) {
+ p->swap_map[offset]++;
+ result = 1;
+ } else if (p->swap_map[offset] <= SWAP_MAP_MAX) {
+ if (swap_overflow++ < 5)
+ printk(KERN_WARNING "swap_dup: swap entry overflow\n");
+ p->swap_map[offset] = SWAP_MAP_MAX;
+ result = 1;
+ }
+ }
+ swap_device_unlock(p);
+out:
+ return result;
+
+bad_file:
+ printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val);
+ goto out;
+}
+
+struct swap_info_struct *
+get_swap_info_struct(unsigned type)
+{
+ return &swap_info[type];
+}
+
+/*
+ * swap_device_lock prevents swap_map being freed. Don't grab an extra
+ * reference on the swaphandle, it doesn't matter if it becomes unused.
+ */
+int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
+{
+ int ret = 0, i = 1 << page_cluster;
+ unsigned long toff;
+ struct swap_info_struct *swapdev = swp_type(entry) + swap_info;
+
+ if (!page_cluster) /* no readahead */
+ return 0;
+ toff = (swp_offset(entry) >> page_cluster) << page_cluster;
+ if (!toff) /* first page is swap header */
+ toff++, i--;
+ *offset = toff;
+
+ swap_device_lock(swapdev);
+ do {
+ /* Don't read-ahead past the end of the swap area */
+ if (toff >= swapdev->max)
+ break;
+ /* Don't read in free or bad pages */
+ if (!swapdev->swap_map[toff])
+ break;
+ if (swapdev->swap_map[toff] == SWAP_MAP_BAD)
+ break;
+ toff++;
+ ret++;
+ } while (--i);
+ swap_device_unlock(swapdev);
+ return ret;
+}
projects / xen.git / commitdiff